benchmarks.yml

name: Benchmarks

# Checks that every benchmark runs on every Ruby (PRs: only the files they change).
# It publishes nothing: the results site comes from results-site.yml.
# Always runs, so the benchmarks-ok check at the end always reports and can be
# required. The changes job decides whether there is anything to benchmark.
on:
  push:
    branches: [ main ]
  pull_request:
    branches: [ main ]

jobs:
  changes:
    runs-on: ubuntu-latest
    outputs:
      run: ${{ steps.pick.outputs.run }}
      files: ${{ steps.pick.outputs.files }}

    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Pick benchmarks
        id: pick
        # A pull_request checks out a merge commit: HEAD^1 is the base.
        run: .github/scripts/pick-benchmarks.sh "${{ github.event_name == 'pull_request' && 'HEAD^1' || github.event.before }}"

  # Every benchmark has the same shape (see lint-benchmarks.rb). Reads the
  # files without running them, on the newest Ruby: the lint needs Prism.
  lint:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4
      - name: Lint benchmarks
        run: docker compose run --rm -T --entrypoint ruby ruby_4.0 .github/scripts/lint-benchmarks.rb

  # Waits for the lint: if it fails, the PR cannot merge, so the benchmark
  # jobs would only hold the org's job slots for nothing.
  rake:
    name: rake (${{ matrix.ruby }}${{ matrix.variant && format('+{0}', matrix.variant) || '' }})
    needs: [changes, lint]
    if: needs.changes.outputs.run == 'true'
    runs-on: ubuntu-latest
    timeout-minutes: 60

    strategy:
      fail-fast: false
      matrix:
        # Service names from compose.yaml
        ruby: [
          'ruby_head', 'ruby_4.0', 'ruby_3.4', 'ruby_3.3', 'ruby_3.2', 'ruby_3.1',
          'ruby_3.0', 'ruby_2.7', 'ruby_2.6', 'ruby_2.5', 'ruby_2.4', 'ruby_2.3',
          'ruby_2.2', 'ruby_2.1',
          'jruby_head', 'jruby_9.1',
          'truffleruby_head', 'truffleruby_22'
        ]
        # '' is the plain Ruby. Each include below overwrites it, so GitHub
        # adds it as a separate job instead of merging it into the plain one.
        variant: ['']
        include:
          - { ruby: ruby_3.1, variant: yjit, flags: --yjit }
          - { ruby: ruby_3.2, variant: yjit, flags: --yjit }
          - { ruby: ruby_3.3, variant: yjit, flags: --yjit }
          - { ruby: ruby_3.4, variant: yjit, flags: --yjit }
          - { ruby: ruby_4.0, variant: yjit, flags: --yjit }
          - { ruby: ruby_head, variant: yjit, flags: --yjit }
          - { ruby: ruby_4.0, variant: zjit, flags: --zjit }
          - { ruby: ruby_head, variant: zjit, flags: --zjit }

    env:
      # Read by docker/run-benchmarks.sh.
      RUBY_VARIANT: ${{ matrix.variant }}
      RUBY_VARIANT_FLAGS: ${{ matrix.flags }}
      # The build's name in the step names and the failure report, for example ruby_3.4+yjit.
      BUILD: ${{ matrix.ruby }}${{ matrix.variant && format('+{0}', matrix.variant) || '' }}

    steps:
      # Plain Rubies only: shared reports do not record the Ruby or its flags,
      # so variant runs would be indistinguishable on ips.fastruby.io.
      - name: Set Share Env
        if: github.ref_name == 'main' && matrix.variant == ''
        run: |
          echo "SHARE=1" >> "$GITHUB_ENV"
      - uses: actions/checkout@v4
      - name: Run benchmarks on ${{ env.BUILD }}
        env:
          FILES: ${{ needs.changes.outputs.files }}
        run: .github/scripts/run-benchmarks.sh ${{ matrix.ruby }}
      - name: Report failed benchmarks
        if: failure()
        run: .github/scripts/report-failures.sh benchmarks.log ${{ env.BUILD }}

  # The check to require on main. Passes when every benchmark job passed, or
  # when there was nothing to benchmark.
  benchmarks-ok:
    needs: [changes, lint, rake]
    if: always()
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4
      - name: Check the benchmark jobs
        run: .github/scripts/check-benchmarks.sh ${{ needs.changes.result }} ${{ needs.lint.result }} ${{ needs.rake.result }}