Add automated retry for network errors in install.sh and CI

.github/workflows/build.yml

@@ -293,41 +293,62 @@ jobs:
       CFLAGS: ${{ matrix.CFLAGS }} ${{ inputs.cflags }}
       FPM_VERSION: ${{ matrix.FPM_VERSION || '0.13.0' }}
       NETWORK_ARG: ${{ matrix.network && format('--network={0}',matrix.network) || '' }}
-      CURL_OPTIONS: -L --retry 10 --retry-all-errors --fail 
+      CURL_OPTIONS: --retry 10 --retry-all-errors --fail -L
       CONFIGURE_ARGS: --prefix=install --enable-rpath --enable-debug --verbose ${{ matrix.CONFIGURE_ARGS }}
       GASNET_SPAWN_VERBOSE: 1
       GASNET_SPAWNFN: L
       GASNET_MASTERIP: 127.0.0.1
       CAF_IMAGES: 8
-      BREW_WARNING_FILTER: 's/^[[:punct:]]*warning[[:punct:]]*(You are using macOS on Intel)/warn: \1/i'
+      BREW_WARNING_FILTER: 's/^[[:punct:]]*warning[[:punct:]]*(You are using macOS)/warn: \1/i'
       INSTALL_VARS: FC CC CXX FFLAGS CPPFLAGS CFLAGS CXXFLAGS LDFLAGS LIBS GASNET_CONFIGURE_ARGS FPM CMAKE MAKE PKG_CONFIG
 
     steps:
     - name: Checkout code
       uses: actions/checkout@v1
 
+    - name: Install shell functions
+      run: |
+        mkdir -p ~/.bash_helpers
+        cat << 'EOF' > ~/.bash_helpers/custom_functions.sh
+        retry() { 
+          local retry_cnt=0
+          local retry_delay
+          while ! "$@" ; do 
+            retry_cnt=$(expr $retry_cnt + 1)
+            if [[ $retry_cnt > 5 ]] ; then exit 1 ; fi 
+            ( set +x ; 
+              echo "::warning::Command failed: <$@> Pausing before retry $retry_cnt..." ); 
+            retry_delay=$(( 5 ** retry_cnt ))
+            if [[ $retry_delay > 600 ]] ; then retry_delay=600 ; fi
+            sleep $retry_delay
+          done 
+        }
+        EOF
+        # Register file in BASH_ENV so it automatically loads in all future steps
+        echo "BASH_ENV=~/.bash_helpers/custom_functions.sh" >> $GITHUB_ENV
+
     ##########################################################################
     # Establish Dependencies
 
     - name: Install Ubuntu Container Dependencies
       if: ${{ contains(matrix.os, 'ubuntu') && matrix.container != '' && !contains(matrix.container, 'phhargrove') }}
       run: |
-        apt update
-        apt install -y build-essential pkg-config make git curl cmake
+        retry apt update
+        retry apt install -y build-essential pkg-config make git curl cmake
 
     - name: Install Ubuntu Native Dependencies
       if: ${{ contains(matrix.os, 'ubuntu') && matrix.container == '' && !matrix.brew_via_install }}
       run: |
-        sudo apt update
-        sudo apt install -y build-essential pkg-config make
+        retry sudo apt update
+        retry sudo apt install -y build-essential pkg-config make
         if [[ ${COMPILER_VERSION} -lt 15 ]] || type -p gfortran-${COMPILER_VERSION} ; then
-          sudo apt install -y gfortran-${COMPILER_VERSION} g++-${COMPILER_VERSION}
+          retry sudo apt install -y gfortran-${COMPILER_VERSION} g++-${COMPILER_VERSION}
         else
           curl $CURL_OPTIONS https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh -o install-homebrew.sh
           chmod +x install-homebrew.sh
           env CI=1 ./install-homebrew.sh
           HOMEBREW_PREFIX="/home/linuxbrew/.linuxbrew"
-          ${HOMEBREW_PREFIX}/bin/brew install -v gcc@${COMPILER_VERSION} binutils
+          retry ${HOMEBREW_PREFIX}/bin/brew install -v gcc@${COMPILER_VERSION} binutils
           ls -al ${HOMEBREW_PREFIX}/bin
           echo "PATH=${HOMEBREW_PREFIX}/bin:${PATH}" >> "$GITHUB_ENV"
           : Homebrew GCC@15 needs binutils 2.44+
@@ -343,7 +364,7 @@ jobs:
       if: ${{ contains(matrix.os, 'macos') && !matrix.macports }}
       run: |
         # silence Homebrew tap trust warnings:
-        (set +e ; brew untap -f aws/tap hashicorp/tap 2>&1 | perl -pe 's/#*.warning./warn: /' ; brew trust azure/bicep )
+        (set +e ; brew untap -f aws/tap hashicorp/tap 2>&1 | perl -pe 's/^::*//;s/^##*//' ; brew trust azure/bicep ; echo )
         # set some Homebrew knobs to accelerate installation
         # but only when we are not simulating the user environment brew-via-install workflow
         if [[ -z "${{ matrix.brew_via_install }}" ]] ; then
@@ -360,12 +381,12 @@ jobs:
     - name: Install macOS Dependencies (Homebrew)
       if: ${{ contains(matrix.os, 'macos') && !matrix.macports && !matrix.brew_via_install }}
       run: |
-        brew update
+        retry brew update
         # fpm binary distribution for macOS requires gfortran shared libraries from gcc@12
-        brew install gcc@12 \
+        retry brew install gcc@12 \
             2>&1 | perl -pe "$BREW_WARNING_FILTER" ; test ${PIPESTATUS[0]} = 0
         if [[ ${{ matrix.compiler }} == 'gfortran' && ${COMPILER_VERSION} -gt 15 ]] ; then
-          brew install gcc@${COMPILER_VERSION} \
+          retry brew install gcc@${COMPILER_VERSION} \
             2>&1 | perl -pe "$BREW_WARNING_FILTER" ; test ${PIPESTATUS[0]} = 0
         fi
 
@@ -374,24 +395,23 @@ jobs:
       run: |
         #sudo port -v selfupdate
         if [[ ${{ matrix.compiler }} == 'gfortran' && ${COMPILER_VERSION} -gt 15 ]] ; then
-          sudo port -v install gcc${COMPILER_VERSION}
+          retry sudo port -v install gcc${COMPILER_VERSION}
         fi
         # Append Macports dynamic library directory
         echo "DYLD_FALLBACK_LIBRARY_PATH=/opt/local/lib/libgcc/:$DYLD_FALLBACK_LIBRARY_PATH" >> "$GITHUB_ENV"
 
     - name: Install LLVM flang on macOS (Homebrew)
       if: ${{ contains(matrix.os, 'macos') && matrix.compiler == 'flang' && !matrix.macports && !matrix.brew_via_install }}
       run: |
-        brew install llvm@${COMPILER_VERSION} flang \
+        retry brew install llvm@${COMPILER_VERSION} flang \
             2>&1 | perl -pe "$BREW_WARNING_FILTER" ; test ${PIPESTATUS[0]} = 0
-          perl -pe "$BREW_WARNING_FILTER" ; test ${PIPESTATUS[0]} = 0
         # Prepend homebrew clang to PATH:
         echo "PATH=$(brew --prefix)/opt/llvm/bin:${PATH}" >> "$GITHUB_ENV"
 
     - name: Install LLVM flang on macOS (Macports)
       if: ${{ matrix.compiler == 'flang' && matrix.macports }}
       run: |
-        sudo port -v install flang-${COMPILER_VERSION}
+        retry sudo port -v install flang-${COMPILER_VERSION}
         sudo ln -sf /opt/local/bin/flang-mp-${COMPILER_VERSION} /opt/local/bin/flang
         sudo ln -sf /opt/local/bin/flang-mp-${COMPILER_VERSION} /opt/local/bin/flang-new
         sudo ln -sf /opt/local/bin/clang-mp-${COMPILER_VERSION} /opt/local/bin/clang
@@ -429,8 +449,8 @@ jobs:
         [[ -n "$GFORTRAN" ]]
         mkdir -p fpm-temp/bin
         cd fpm-temp
-        curl --retry 5 -LOsS https://github.com/fortran-lang/fpm/releases/download/v$FPM_VERSION/fpm-$FPM_VERSION.F90
-        $GFORTRAN -fopenmp -o bin/fpm fpm-$FPM_VERSION.F90
+        curl $CURL_OPTIONS https://github.com/fortran-lang/fpm/releases/download/v$FPM_VERSION/fpm-$FPM_VERSION.F90 -o fpm.F90
+        $GFORTRAN -fopenmp -o bin/fpm fpm.F90
         echo "PATH=$PWD/bin:${PATH}" >> "$GITHUB_ENV"
 
     ##########################################################################
@@ -463,8 +483,8 @@ jobs:
     - name: Setup lfortran dependencies and variables
       if: ${{ matrix.compiler == 'lfortran' }}
       run: |
-        apt update
-        apt install -y llvm clang
+        retry apt update
+        retry apt install -y llvm clang
         if [ ${{ contains(matrix.container, 'ghcr.io/lfortran') }} = true ]; then
           ls -alR /app 
           # Add container lfortran to PATH:

install.sh

@@ -149,6 +149,20 @@ realpath() {
     ' "$1"
 }
 
+retry() { 
+  local retry_cnt=0
+  local retry_delay
+  while ! "$@" ; do
+    retry_cnt=$(expr $retry_cnt + 1)
+    if [[ $retry_cnt > 5 ]] ; then return 1 ; fi
+    ( set +x ; 
+      echo "::warning::Command failed: <$@> Pausing before retry $retry_cnt..." );
+    retry_delay=$(( 5 ** retry_cnt ))
+    if [[ $retry_delay > 600 ]] ; then retry_delay=600 ; fi
+    sleep $retry_delay
+  done 
+}
+
 append_gasnet_configure_arg() {
   if [[ -z "$GASNET_CONFIGURE_ARGS" ]] ; then
     GASNET_CONFIGURE_ARGS="\"$1\""
@@ -406,7 +420,7 @@ if [ -z ${FC:+x} ] || [ -z ${CC:+x} ] || [ -z ${PKG_CONFIG:+x} ] || [ -z ${MAKE:
     ask_permission_to_install_homebrew
     exit_if_user_declines "brew"
 
-    $CURL -L https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh -o $DEPENDENCIES_DIR/install-homebrew.sh --create-dirs
+    $CURL --retry 10 --retry-all-errors --fail -L https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh -o $DEPENDENCIES_DIR/install-homebrew.sh --create-dirs
     chmod u+x $DEPENDENCIES_DIR/install-homebrew.sh
 
     if [ -p /dev/stdin ] && [ $CI = false ]; then
@@ -437,12 +451,12 @@ EOF
   fi
 
   # fetch the latest package definitions:
-  $BREW update
+  retry $BREW update
 
   if [ -z ${FC:+x} ] || [ -z ${CC:+x} ] ; then
     ask_permission_to_install_homebrew_package "'llvm' and 'flang'"
     exit_if_user_declines "FC"
-    $BREW install llvm flang
+    retry $BREW install llvm flang
 
     # Homebrew does not inject clang/clang++ into PATH on macOS
     export PATH="$BREW_PREFIX/opt/llvm/bin:$PATH"
@@ -462,28 +476,28 @@ EOF
   if [ -z ${MAKE:+x} ] ; then
     ask_permission_to_install_homebrew_package "'make'"
     exit_if_user_declines "make"
-    $BREW install make
+    retry $BREW install make
     MAKE=$(abswhich gmake)
   fi
 
   if [ -z ${PKG_CONFIG:+x} ]; then
     ask_permission_to_install_homebrew_package "'pkg-config'"
     exit_if_user_declines "pkg-config"
-    $BREW install pkg-config
+    retry $BREW install pkg-config
     PKG_CONFIG=$(abswhich pkg-config)
   fi
 
   if [ -z ${FPM:+x} ] ; then
     ask_permission_to_install_homebrew_package "'fpm'"
     exit_if_user_declines "fpm"
-    $BREW install fpm
+    retry $BREW install fpm
     FPM=$(abswhich fpm)
   fi
 
   if [ -z ${CMAKE:+x} -a -z ${USE_FPM:+x} ] ; then
     ask_permission_to_install_homebrew_package "'cmake'"
     exit_if_user_declines "cmake"
-    $BREW install cmake
+    retry $BREW install cmake
     CMAKE=$(abswhich cmake)
   fi
 fi
@@ -632,7 +646,7 @@ if ! $PKG_CONFIG $pkg ; then
     rm -Rf $GASNET_DIR
   fi
 
-  $CURL -L $VERBOSE --retry 10 --retry-all-errors --fail $GASNET_SOURCE_URL -o $GASNET_TAR_FILE
+  $CURL $VERBOSE --retry 10 --retry-all-errors --fail -L $GASNET_SOURCE_URL -o $GASNET_TAR_FILE
   tar xvzf $GASNET_TAR_FILE -C $DEPENDENCIES_DIR
 
   ( 
@@ -914,6 +928,29 @@ error_handler() {
 
 trap 'error_handler $LINENO' ERR
 
+# ---------------------------------------------------------------
+# Pre-fetch dependencies, with retries for resilience to transient network failures
+# This is required for CMake, and optional but recommended for FPM
+
+ASSERT_DIR=$DEPENDENCIES_DIR/assert
+JULIENNE_DIR=$DEPENDENCIES_DIR/julienne
+
+PREFETCH_DEPS=${PREFETCH_DEPS:-true}
+[[ -z $USE_FPM ]] && PREFETCH_DEPS=true
+
+if [[ $PREFETCH_DEPS == "true" ]] ; then
+  # Git commands assume git version 1.7.7 (2011-09) or later
+  mkdir -p $ASSERT_DIR
+  if ! [[ -r $ASSERT_DIR/fpm.toml ]] ; then
+    retry $GIT clone -c advice.detachedHead=false --depth 1 --branch $ASSERT_VERSION $ASSERT_GIT $ASSERT_DIR
+    ( cd $ASSERT_DIR && $GIT log -n 1 --oneline )
+  fi
+  if [[ -n $USE_FPM ]] && ! [[ -r $JULIENNE_DIR/fpm.toml ]] ; then
+    retry $GIT clone -c advice.detachedHead=false --depth 1 --branch $JULIENNE_VERSION $JULIENNE_GIT $JULIENNE_DIR
+    ( cd $JULIENNE_DIR && $GIT log -n 1 --oneline )
+  fi
+fi
+
 # ---------------------------------------------------------------
 # Caffeine build
 
@@ -926,13 +963,6 @@ if [[ -n $USE_FPM ]] ; then
 
   LIBCAFFEINE_SRC=$(./$RUN_FPM_SH install --list 2>/dev/null | grep libcaffeine | cut -d' ' -f2)
 else # Using CMake instead of FPM to build
-  ASSERT_DIR=$DEPENDENCIES_DIR/assert
-  mkdir -p $ASSERT_DIR
-  if ! [[ -r $ASSERT_DIR/fpm.toml ]] ; then
-    # Download Assert: Assumes git version 1.7.7 (2011-09) or later
-    $GIT clone -c advice.detachedHead=false --depth 1 --branch $ASSERT_VERSION $ASSERT_GIT $ASSERT_DIR
-    ( cd $ASSERT_DIR && $GIT log -n 1 --oneline )
-  fi
 
   # CMake botches module name analysis unless we match the name in the source file:
   ASSERT_SRC="$ASSERT_DIR/src/caf_caffiene_assert_m.F90"