diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e795d509..9ed10e09 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -293,41 +293,62 @@ jobs: CFLAGS: ${{ matrix.CFLAGS }} ${{ inputs.cflags }} FPM_VERSION: ${{ matrix.FPM_VERSION || '0.13.0' }} NETWORK_ARG: ${{ matrix.network && format('--network={0}',matrix.network) || '' }} - CURL_OPTIONS: -L --retry 10 --retry-all-errors --fail + CURL_OPTIONS: --retry 10 --retry-all-errors --fail -L CONFIGURE_ARGS: --prefix=install --enable-rpath --enable-debug --verbose ${{ matrix.CONFIGURE_ARGS }} GASNET_SPAWN_VERBOSE: 1 GASNET_SPAWNFN: L GASNET_MASTERIP: 127.0.0.1 CAF_IMAGES: 8 - BREW_WARNING_FILTER: 's/^[[:punct:]]*warning[[:punct:]]*(You are using macOS on Intel)/warn: \1/i' + BREW_WARNING_FILTER: 's/^[[:punct:]]*warning[[:punct:]]*(You are using macOS)/warn: \1/i' INSTALL_VARS: FC CC CXX FFLAGS CPPFLAGS CFLAGS CXXFLAGS LDFLAGS LIBS GASNET_CONFIGURE_ARGS FPM CMAKE MAKE PKG_CONFIG steps: - name: Checkout code uses: actions/checkout@v1 + - name: Install shell functions + run: | + mkdir -p ~/.bash_helpers + cat << 'EOF' > ~/.bash_helpers/custom_functions.sh + retry() { + local retry_cnt=0 + local retry_delay + while ! "$@" ; do + retry_cnt=$(expr $retry_cnt + 1) + if [[ $retry_cnt > 5 ]] ; then exit 1 ; fi + ( set +x ; + echo "::warning::Command failed: <$@> Pausing before retry $retry_cnt..." ); + retry_delay=$(( 5 ** retry_cnt )) + if [[ $retry_delay > 600 ]] ; then retry_delay=600 ; fi + sleep $retry_delay + done + } + EOF + # Register file in BASH_ENV so it automatically loads in all future steps + echo "BASH_ENV=~/.bash_helpers/custom_functions.sh" >> $GITHUB_ENV + ########################################################################## # Establish Dependencies - name: Install Ubuntu Container Dependencies if: ${{ contains(matrix.os, 'ubuntu') && matrix.container != '' && !contains(matrix.container, 'phhargrove') }} run: | - apt update - apt install -y build-essential pkg-config make git curl cmake + retry apt update + retry apt install -y build-essential pkg-config make git curl cmake - name: Install Ubuntu Native Dependencies if: ${{ contains(matrix.os, 'ubuntu') && matrix.container == '' && !matrix.brew_via_install }} run: | - sudo apt update - sudo apt install -y build-essential pkg-config make + retry sudo apt update + retry sudo apt install -y build-essential pkg-config make if [[ ${COMPILER_VERSION} -lt 15 ]] || type -p gfortran-${COMPILER_VERSION} ; then - sudo apt install -y gfortran-${COMPILER_VERSION} g++-${COMPILER_VERSION} + retry sudo apt install -y gfortran-${COMPILER_VERSION} g++-${COMPILER_VERSION} else curl $CURL_OPTIONS https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh -o install-homebrew.sh chmod +x install-homebrew.sh env CI=1 ./install-homebrew.sh HOMEBREW_PREFIX="/home/linuxbrew/.linuxbrew" - ${HOMEBREW_PREFIX}/bin/brew install -v gcc@${COMPILER_VERSION} binutils + retry ${HOMEBREW_PREFIX}/bin/brew install -v gcc@${COMPILER_VERSION} binutils ls -al ${HOMEBREW_PREFIX}/bin echo "PATH=${HOMEBREW_PREFIX}/bin:${PATH}" >> "$GITHUB_ENV" : Homebrew GCC@15 needs binutils 2.44+ @@ -343,7 +364,7 @@ jobs: if: ${{ contains(matrix.os, 'macos') && !matrix.macports }} run: | # silence Homebrew tap trust warnings: - (set +e ; brew untap -f aws/tap hashicorp/tap 2>&1 | perl -pe 's/#*.warning./warn: /' ; brew trust azure/bicep ) + (set +e ; brew untap -f aws/tap hashicorp/tap 2>&1 | perl -pe 's/^::*//;s/^##*//' ; brew trust azure/bicep ; echo ) # set some Homebrew knobs to accelerate installation # but only when we are not simulating the user environment brew-via-install workflow if [[ -z "${{ matrix.brew_via_install }}" ]] ; then @@ -360,12 +381,12 @@ jobs: - name: Install macOS Dependencies (Homebrew) if: ${{ contains(matrix.os, 'macos') && !matrix.macports && !matrix.brew_via_install }} run: | - brew update + retry brew update # fpm binary distribution for macOS requires gfortran shared libraries from gcc@12 - brew install gcc@12 \ + retry brew install gcc@12 \ 2>&1 | perl -pe "$BREW_WARNING_FILTER" ; test ${PIPESTATUS[0]} = 0 if [[ ${{ matrix.compiler }} == 'gfortran' && ${COMPILER_VERSION} -gt 15 ]] ; then - brew install gcc@${COMPILER_VERSION} \ + retry brew install gcc@${COMPILER_VERSION} \ 2>&1 | perl -pe "$BREW_WARNING_FILTER" ; test ${PIPESTATUS[0]} = 0 fi @@ -374,7 +395,7 @@ jobs: run: | #sudo port -v selfupdate if [[ ${{ matrix.compiler }} == 'gfortran' && ${COMPILER_VERSION} -gt 15 ]] ; then - sudo port -v install gcc${COMPILER_VERSION} + retry sudo port -v install gcc${COMPILER_VERSION} fi # Append Macports dynamic library directory echo "DYLD_FALLBACK_LIBRARY_PATH=/opt/local/lib/libgcc/:$DYLD_FALLBACK_LIBRARY_PATH" >> "$GITHUB_ENV" @@ -382,16 +403,15 @@ jobs: - name: Install LLVM flang on macOS (Homebrew) if: ${{ contains(matrix.os, 'macos') && matrix.compiler == 'flang' && !matrix.macports && !matrix.brew_via_install }} run: | - brew install llvm@${COMPILER_VERSION} flang \ + retry brew install llvm@${COMPILER_VERSION} flang \ 2>&1 | perl -pe "$BREW_WARNING_FILTER" ; test ${PIPESTATUS[0]} = 0 - perl -pe "$BREW_WARNING_FILTER" ; test ${PIPESTATUS[0]} = 0 # Prepend homebrew clang to PATH: echo "PATH=$(brew --prefix)/opt/llvm/bin:${PATH}" >> "$GITHUB_ENV" - name: Install LLVM flang on macOS (Macports) if: ${{ matrix.compiler == 'flang' && matrix.macports }} run: | - sudo port -v install flang-${COMPILER_VERSION} + retry sudo port -v install flang-${COMPILER_VERSION} sudo ln -sf /opt/local/bin/flang-mp-${COMPILER_VERSION} /opt/local/bin/flang sudo ln -sf /opt/local/bin/flang-mp-${COMPILER_VERSION} /opt/local/bin/flang-new sudo ln -sf /opt/local/bin/clang-mp-${COMPILER_VERSION} /opt/local/bin/clang @@ -429,8 +449,8 @@ jobs: [[ -n "$GFORTRAN" ]] mkdir -p fpm-temp/bin cd fpm-temp - curl --retry 5 -LOsS https://github.com/fortran-lang/fpm/releases/download/v$FPM_VERSION/fpm-$FPM_VERSION.F90 - $GFORTRAN -fopenmp -o bin/fpm fpm-$FPM_VERSION.F90 + curl $CURL_OPTIONS https://github.com/fortran-lang/fpm/releases/download/v$FPM_VERSION/fpm-$FPM_VERSION.F90 -o fpm.F90 + $GFORTRAN -fopenmp -o bin/fpm fpm.F90 echo "PATH=$PWD/bin:${PATH}" >> "$GITHUB_ENV" ########################################################################## @@ -463,8 +483,8 @@ jobs: - name: Setup lfortran dependencies and variables if: ${{ matrix.compiler == 'lfortran' }} run: | - apt update - apt install -y llvm clang + retry apt update + retry apt install -y llvm clang if [ ${{ contains(matrix.container, 'ghcr.io/lfortran') }} = true ]; then ls -alR /app # Add container lfortran to PATH: diff --git a/install.sh b/install.sh index c437d472..9db2858d 100755 --- a/install.sh +++ b/install.sh @@ -149,6 +149,20 @@ realpath() { ' "$1" } +retry() { + local retry_cnt=0 + local retry_delay + while ! "$@" ; do + retry_cnt=$(expr $retry_cnt + 1) + if [[ $retry_cnt > 5 ]] ; then return 1 ; fi + ( set +x ; + echo "::warning::Command failed: <$@> Pausing before retry $retry_cnt..." ); + retry_delay=$(( 5 ** retry_cnt )) + if [[ $retry_delay > 600 ]] ; then retry_delay=600 ; fi + sleep $retry_delay + done +} + append_gasnet_configure_arg() { if [[ -z "$GASNET_CONFIGURE_ARGS" ]] ; then GASNET_CONFIGURE_ARGS="\"$1\"" @@ -406,7 +420,7 @@ if [ -z ${FC:+x} ] || [ -z ${CC:+x} ] || [ -z ${PKG_CONFIG:+x} ] || [ -z ${MAKE: ask_permission_to_install_homebrew exit_if_user_declines "brew" - $CURL -L https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh -o $DEPENDENCIES_DIR/install-homebrew.sh --create-dirs + $CURL --retry 10 --retry-all-errors --fail -L https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh -o $DEPENDENCIES_DIR/install-homebrew.sh --create-dirs chmod u+x $DEPENDENCIES_DIR/install-homebrew.sh if [ -p /dev/stdin ] && [ $CI = false ]; then @@ -437,12 +451,12 @@ EOF fi # fetch the latest package definitions: - $BREW update + retry $BREW update if [ -z ${FC:+x} ] || [ -z ${CC:+x} ] ; then ask_permission_to_install_homebrew_package "'llvm' and 'flang'" exit_if_user_declines "FC" - $BREW install llvm flang + retry $BREW install llvm flang # Homebrew does not inject clang/clang++ into PATH on macOS export PATH="$BREW_PREFIX/opt/llvm/bin:$PATH" @@ -462,28 +476,28 @@ EOF if [ -z ${MAKE:+x} ] ; then ask_permission_to_install_homebrew_package "'make'" exit_if_user_declines "make" - $BREW install make + retry $BREW install make MAKE=$(abswhich gmake) fi if [ -z ${PKG_CONFIG:+x} ]; then ask_permission_to_install_homebrew_package "'pkg-config'" exit_if_user_declines "pkg-config" - $BREW install pkg-config + retry $BREW install pkg-config PKG_CONFIG=$(abswhich pkg-config) fi if [ -z ${FPM:+x} ] ; then ask_permission_to_install_homebrew_package "'fpm'" exit_if_user_declines "fpm" - $BREW install fpm + retry $BREW install fpm FPM=$(abswhich fpm) fi if [ -z ${CMAKE:+x} -a -z ${USE_FPM:+x} ] ; then ask_permission_to_install_homebrew_package "'cmake'" exit_if_user_declines "cmake" - $BREW install cmake + retry $BREW install cmake CMAKE=$(abswhich cmake) fi fi @@ -632,7 +646,7 @@ if ! $PKG_CONFIG $pkg ; then rm -Rf $GASNET_DIR fi - $CURL -L $VERBOSE --retry 10 --retry-all-errors --fail $GASNET_SOURCE_URL -o $GASNET_TAR_FILE + $CURL $VERBOSE --retry 10 --retry-all-errors --fail -L $GASNET_SOURCE_URL -o $GASNET_TAR_FILE tar xvzf $GASNET_TAR_FILE -C $DEPENDENCIES_DIR ( @@ -914,6 +928,29 @@ error_handler() { trap 'error_handler $LINENO' ERR +# --------------------------------------------------------------- +# Pre-fetch dependencies, with retries for resilience to transient network failures +# This is required for CMake, and optional but recommended for FPM + +ASSERT_DIR=$DEPENDENCIES_DIR/assert +JULIENNE_DIR=$DEPENDENCIES_DIR/julienne + +PREFETCH_DEPS=${PREFETCH_DEPS:-true} +[[ -z $USE_FPM ]] && PREFETCH_DEPS=true + +if [[ $PREFETCH_DEPS == "true" ]] ; then + # Git commands assume git version 1.7.7 (2011-09) or later + mkdir -p $ASSERT_DIR + if ! [[ -r $ASSERT_DIR/fpm.toml ]] ; then + retry $GIT clone -c advice.detachedHead=false --depth 1 --branch $ASSERT_VERSION $ASSERT_GIT $ASSERT_DIR + ( cd $ASSERT_DIR && $GIT log -n 1 --oneline ) + fi + if [[ -n $USE_FPM ]] && ! [[ -r $JULIENNE_DIR/fpm.toml ]] ; then + retry $GIT clone -c advice.detachedHead=false --depth 1 --branch $JULIENNE_VERSION $JULIENNE_GIT $JULIENNE_DIR + ( cd $JULIENNE_DIR && $GIT log -n 1 --oneline ) + fi +fi + # --------------------------------------------------------------- # Caffeine build @@ -926,13 +963,6 @@ if [[ -n $USE_FPM ]] ; then LIBCAFFEINE_SRC=$(./$RUN_FPM_SH install --list 2>/dev/null | grep libcaffeine | cut -d' ' -f2) else # Using CMake instead of FPM to build - ASSERT_DIR=$DEPENDENCIES_DIR/assert - mkdir -p $ASSERT_DIR - if ! [[ -r $ASSERT_DIR/fpm.toml ]] ; then - # Download Assert: Assumes git version 1.7.7 (2011-09) or later - $GIT clone -c advice.detachedHead=false --depth 1 --branch $ASSERT_VERSION $ASSERT_GIT $ASSERT_DIR - ( cd $ASSERT_DIR && $GIT log -n 1 --oneline ) - fi # CMake botches module name analysis unless we match the name in the source file: ASSERT_SRC="$ASSERT_DIR/src/caf_caffiene_assert_m.F90"