robot-tests.yml

name: Robot Framework Tests

# One suite, run against a service profile (tests/profiles.yml).
#
# There is deliberately no "with API keys" / "without API keys" split. Every test
# runs in every profile; the stub profile replays recorded real provider
# responses (tests/cassettes/) so content assertions mean the same thing either
# way. What varies between profiles is which backing services are real, not which
# tests execute.
#
#   pull_request      -> mock. Needs no secrets, so it works unchanged on forks.
#                        (A fork PR cannot read secrets under `pull_request` no
#                        matter what label it carries, so the old
#                        label-triggered workflow could never have worked for
#                        the fork case it was added for. Making it work would
#                        need `pull_request_target`, which runs untrusted code
#                        with secrets in scope -- not worth it.)
#   push dev/main     -> mock.
#   workflow_dispatch -> any profile, on demand.
#
# No cloud provider runs automatically. Automatic runs against a paid API cost
# money on every push and pin CI to whichever vendor happens to be listed here,
# which goes stale the moment the deployment switches providers. Testing a
# specific provider's integration is a deliberate act: dispatch the profile you
# want.

on:
  pull_request:
    paths:
      - 'tests/**'
      - 'backend/src/**'
      - '.github/workflows/robot-tests.yml'
  push:
    branches: [dev, main]
    paths:
      - 'tests/**'
      - 'backend/src/**'
      - '.github/workflows/robot-tests.yml'
  workflow_dispatch:
    inputs:
      profile:
        description: 'Service profile to run against'
        required: true
        default: 'mock'
        type: choice
        options:
          - mock
          - smallest-openai
          - deepgram-openai
          - deepgram-openai-speaker
          - parakeet-ollama

permissions:
  contents: read
  pull-requests: write

jobs:
  select-profiles:
    runs-on: ubuntu-latest
    outputs:
      profiles: ${{ steps.pick.outputs.profiles }}
    steps:
      - id: pick
        run: |
          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
            echo 'profiles=["${{ inputs.profile }}"]' >> "$GITHUB_OUTPUT"
          else
            echo 'profiles=["mock"]' >> "$GITHUB_OUTPUT"
          fi

  robot-tests:
    needs: select-profiles
    runs-on: ubuntu-latest
    timeout-minutes: 45
    strategy:
      fail-fast: false
      matrix:
        profile: ${{ fromJSON(needs.select-profiles.outputs.profiles) }}

    steps:
      - uses: actions/checkout@v4

      - uses: docker/setup-buildx-action@v3
        with:
          driver-opts: |
            image=moby/buildkit:latest
            network=host

      - uses: actions/cache@v4
        with:
          path: /tmp/.buildx-cache
          key: ${{ runner.os }}-buildx-${{ hashFiles('backend/Dockerfile', 'backend/pyproject.toml') }}
          restore-keys: ${{ runner.os }}-buildx-

      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - uses: astral-sh/setup-uv@v4
        with:
          version: "latest"

      - name: Create plugins.yml from template
        run: cp config/plugins.yml.template config/plugins.yml

      - name: Run Robot Framework tests (profile=${{ matrix.profile }})
        working-directory: tests
        env:
          PROFILE: ${{ matrix.profile }}
          # Every provider credential the dispatchable profiles might ask for.
          # All are absent for the mock profile and on fork PRs, which is fine --
          # resolve_profile.py fails fast with an actionable message when the
          # chosen profile needs one and it is missing, rather than letting auth
          # errors surface as confusing test failures.
          SMALLEST_API_KEY: ${{ secrets.SMALLEST_API_KEY }}
          DEEPGRAM_API_KEY: ${{ secrets.DEEPGRAM_API_KEY }}
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          HF_TOKEN: ${{ secrets.HF_TOKEN }}
          CLEANUP_CONTAINERS: "false"
        run: |
          TEST_EXIT_CODE=0
          make test OUTPUTDIR=results || TEST_EXIT_CODE=$?
          echo "test_exit_code=$TEST_EXIT_CODE" >> "$GITHUB_ENV"
          exit 0

      - name: Save service logs
        if: always()
        working-directory: tests
        run: |
          mkdir -p logs
          for svc in chronicle-backend-test workers-test mongo-test redis-test; do
            docker logs "backend-test-$svc-1" > "logs/$svc.log" 2>&1 || true
          done

      - name: Upload service logs
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: robot-service-logs-${{ matrix.profile }}
          path: tests/logs/
          retention-days: 7

      - name: Upload Robot reports
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: robot-reports-${{ matrix.profile }}
          path: |
            tests/results/report.html
            tests/results/log.html
            tests/results/output.xml
          retention-days: 14

      - name: Summarize
        if: always()
        working-directory: tests
        run: |
          {
            echo "## Robot Framework — profile \`${{ matrix.profile }}\`"
            echo ""
            if [ -f results/output.xml ]; then
              uv run --with-requirements test-requirements.txt python show_results.py || true
            else
              echo "No results produced."
            fi
          } >> "$GITHUB_STEP_SUMMARY"

      - name: Fail if tests failed
        if: always()
        run: |
          if [ "${test_exit_code:-0}" != "0" ]; then
            echo "Robot Framework tests failed (exit ${test_exit_code})"
            exit 1
          fi