diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index a1b581c6..bcfbed34 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -23,8 +23,22 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} fetch-depth: 0 + - name: Check for Skip Version Flag + id: check_skip + run: | + # Check if the latest commit message contains [skip-version] flag + # This is used to prevent metrics-only commits from triggering version bumps + LATEST_COMMIT=$(git log -1 --format=%B) + if echo "$LATEST_COMMIT" | grep -q "\[skip-version\]"; then + echo "skip=true" >> $GITHUB_OUTPUT + echo "Metrics-only commit detected. Skipping version bump." + else + echo "skip=false" >> $GITHUB_OUTPUT + fi + - name: Read and Increment Version id: version + if: steps.check_skip.outputs.skip != 'true' run: | # Check if VERSION.txt exists if [ ! -f _includes/VERSION.txt ]; then @@ -47,6 +61,7 @@ jobs: echo "version=$NEW_VERSION" >> $GITHUB_OUTPUT - name: Commit Version File + if: steps.check_skip.outputs.skip != 'true' run: | git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" @@ -61,6 +76,7 @@ jobs: fi - name: Create Git Tag + if: steps.check_skip.outputs.skip != 'true' run: | # Check if tag already exists (could happen with concurrent runs) if git rev-parse "v${{ steps.version.outputs.version }}" >/dev/null 2>&1; then diff --git a/.github/workflows/lighthouse-audit.yml b/.github/workflows/lighthouse-audit.yml new file mode 100644 index 00000000..85f43172 --- /dev/null +++ b/.github/workflows/lighthouse-audit.yml @@ -0,0 +1,142 @@ +name: Lighthouse Audit + +on: + push: + tags: + - 'v*' + workflow_dispatch: + inputs: + version: + description: 'Version to audit (optional; uses git tag then VERSION.txt if empty)' + required: false + type: string + +permissions: + contents: write + +jobs: + lighthouse-audit: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + fetch-depth: 0 + + - name: Setup Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: '3.2' + bundler-cache: true + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Install Lighthouse CLI + run: npm install -g lighthouse + + - name: Build Jekyll site + run: bundle exec jekyll build + + - name: Start local server + run: | + cd _site + python3 -m http.server 4000 > /tmp/server.log 2>&1 & + echo $! > /tmp/server.pid + echo "Server starting with PID $(cat /tmp/server.pid)" + # Readiness polling handled by 'Wait for server to be ready' step + + - name: Wait for server to be ready + run: | + # Wait up to 30 seconds for the local HTTP server to become ready + # This ensures Lighthouse can connect before attempting audit + for i in {1..30}; do + if curl -s http://localhost:4000/ > /dev/null 2>&1; then + echo "Server is ready" + exit 0 + fi + echo "Waiting for server... ($i/30)" + sleep 1 + done + echo "Server failed to start within 30 seconds" + echo "=== Server Log ===" + cat /tmp/server.log || echo "(no log available)" + exit 1 + + - name: Determine version + id: version + run: | + # Version resolution strategy (in order of precedence): + # 1. Manual dispatch input (--version parameter) if provided + # 2. Git tag name (v*) if triggered by tag push + # 3. VERSION.txt file as fallback for manual dispatch without explicit version + # The resolved version is passed to tools/lighthouse_audit.rb via --version flag. + VERSION="" + + # If manual dispatch input provided, use it + if [ -n "${{ inputs.version }}" ]; then + VERSION="${{ inputs.version }}" + echo "Using provided version: $VERSION" + else + # Try to extract version from git tag (e.g., v1631 -> 1631) + TAG=$(git describe --tags --exact-match 2>/dev/null) + if [ -n "$TAG" ]; then + # Validate tag matches pattern: starts with 'v' followed by digits + if echo "$TAG" | grep -qE '^v[0-9]+$'; then + VERSION=$(echo "$TAG" | sed 's/^v//') + echo "Extracted version from git tag: $VERSION" + else + echo "Warning: Tag '$TAG' does not match expected pattern (v*), trying fallback" + fi + fi + + # Fallback to VERSION.txt if version not yet determined + if [ -z "$VERSION" ]; then + if [ -f _includes/VERSION.txt ] && [ -s _includes/VERSION.txt ]; then + VERSION=$(cat _includes/VERSION.txt) + echo "Using version from VERSION.txt: $VERSION" + fi + fi + fi + + # Validate version is not empty + if [ -z "$VERSION" ]; then + echo "ERROR: Could not determine version from:" + echo " 1. Manual workflow input (--version flag)" + echo " 2. Git tag (v* pattern)" + echo " 3. _includes/VERSION.txt file" + echo "Please ensure at least one of these sources is available." + exit 1 + fi + + echo "version=$VERSION" >> $GITHUB_OUTPUT + + - name: Run Lighthouse audit + run: | + ruby tools/lighthouse_audit.rb \ + --version "${{ steps.version.outputs.version }}" \ + --site-url http://localhost:4000 + + - name: Stop local server + if: always() + run: | + if [ -f /tmp/server.pid ]; then + kill $(cat /tmp/server.pid) || true + fi + + - name: Commit quality metrics + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add _data/quality_log.yml + + # Only commit if there are changes + if git diff --staged --quiet; then + echo "No changes to commit" + else + git commit -m "chore: record Lighthouse metrics for v${{ steps.version.outputs.version }} [skip-version]" + git push + fi diff --git a/ROADMAP.md b/ROADMAP.md index 9f3fb240..235288c3 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -22,7 +22,7 @@ The July 2026 readiness assessment found the repository ready for supervised con ## Phase 2: Quality Ledger & Metrics *Goal: Associate every site version with a specific quality snapshot.* -- [ ] **Lighthouse Tracking**: Automate Lighthouse audits during CI and record Performance, Accessibility, and SEO scores per version. +- [x] **Lighthouse Tracking**: Automate Lighthouse audits during CI and record Performance, Accessibility, and SEO scores per version. - [ ] **Link Integrity**: Implement a broken link checker (e.g., `linkinator`) to log broken link counts against the current version. - [ ] **Build Analytics**: Track and log build times to monitor the impact of site growth on CI/CD performance. - [x] **Bug Attribution**: Update Issue Templates to include a "Site Version" field to track bug counts relative to specific releases. @@ -38,6 +38,29 @@ The July 2026 readiness assessment found the repository ready for supervised con - Treat the issue form's Site Version field as the collection mechanism for bug attribution; aggregate reporting belongs in Phase 4. - Update the README when the ledger exists and validate the workflow manually before relying on tag-triggered collection. +### Implementation: Lighthouse Tracking (Completed) + +**Files added/modified:** +- `.github/workflows/lighthouse-audit.yml`: Workflow triggered on release tags (`v*`) or manual dispatch. Builds Jekyll site, serves locally, runs Lighthouse CLI, and commits results to `_data/quality_log.yml` with `[skip-version]` flag. +- `tools/lighthouse_audit.rb`: Ruby script that runs Lighthouse, parses JSON output, extracts Performance/Accessibility/SEO scores, and appends to quality log in YAML format. +- `_data/quality_log.yml`: Append-only quality metrics log with stable YAML format. +- `.github/workflows/deploy.yml`: Updated to check for `[skip-version]` commit flag and skip version bump when metrics-only commit is detected. + +**How it works:** +1. Lighthouse audit workflow is triggered by release tag creation (via `deploy.yml`) or manual workflow dispatch. +2. Workflow checks out code, builds Jekyll site, and starts a local HTTP server. +3. `tools/lighthouse_audit.rb` runs Lighthouse CLI against localhost and parses results. +4. Scores are appended to `_data/quality_log.yml` with version, release date, commit SHA, and workflow run URL. +5. Changes are committed with `[skip-version]` flag to prevent cascading version bumps. +6. `deploy.yml` detects the flag and skips version increment, preventing CI/CD loop. + +**Design decisions:** +- Local server instead of production: Ensures repeatable, controlled audits without external dependencies. Python's built-in `http.server` module is used to serve the Jekyll-built site because it's lightweight, included in all standard CI runners, and requires no additional dependencies beyond what's already available. +- Ruby script for audit coordination: The audit script is written in Ruby (matching the repository's existing tooling in `tools/`) to handle Lighthouse CLI execution, JSON parsing, and quality log updates. It receives the site URL from the workflow and focuses on metric collection. +- Append-only format: Preserves historical data for trend analysis (Phase 4). +- `[skip-version]` commit flag: Prevents metrics collection from disrupting versioning workflow. +- YAML format: Consistent with existing site configuration, human-readable for inspection. + Changes to `.github/workflows/deploy.yml`, release tags, Pages deployment, or versioning require maintainer approval before implementation. ## Phase 3: Developer Experience (DX) & AI Workflows diff --git a/_data/quality_log.yml b/_data/quality_log.yml new file mode 100644 index 00000000..65fa925c --- /dev/null +++ b/_data/quality_log.yml @@ -0,0 +1,17 @@ +# Quality Log - Lighthouse Audit Metrics +# Stores performance, accessibility, and SEO scores for each released version. +# Format: stable YAML with append-only entries, one per released version. +# Version field format: integer (e.g., 1631). +# Scores: 0-100 integer scale (100 is best). +# +# Structure: +# - version: +# release_date: +# commit_sha: +# performance: <0-100> +# accessibility: <0-100> +# seo: <0-100> +# workflow_run_url: + +# This file is updated automatically by the lighthouse-audit workflow. +# Do not edit manually; append entries only via the automated workflow. diff --git a/tools/lighthouse_audit.rb b/tools/lighthouse_audit.rb new file mode 100755 index 00000000..72d152f8 --- /dev/null +++ b/tools/lighthouse_audit.rb @@ -0,0 +1,184 @@ +#!/usr/bin/env ruby +# tools/lighthouse_audit.rb +# Runs Lighthouse audits against the built site and records metrics. +# +# Usage: +# ruby tools/lighthouse_audit.rb [--version VERSION] [--site-url URL] +# +# Environment variables: +# GITHUB_ACTIONS - Set by GitHub Actions to indicate running in CI +# GITHUB_SHA - Commit SHA (GitHub Actions) +# GITHUB_RUN_ID - Workflow run ID (GitHub Actions) +# GITHUB_SERVER_URL - GitHub server URL (GitHub Actions) +# GITHUB_REPOSITORY - Repository name (GitHub Actions) + +require 'json' +require 'date' +require 'yaml' +require 'optparse' + +# Parse command-line arguments +options = {} +OptionParser.new do |opts| + opts.on('--version VERSION', String, 'Version number to record') do |v| + options[:version] = v + end + opts.on('--site-url URL', String, 'URL to audit (default: http://localhost:4000)') do |v| + options[:site_url] = v + end +end.parse! + +# Default values +site_url = options[:site_url] || 'http://localhost:4000' +version = options[:version] || ENV['VERSION'] +commit_sha = ENV['GITHUB_SHA'] || `git rev-parse HEAD`.strip +workflow_run_url = nil + +# Validate required parameters +if version.nil? || version.empty? + puts "ERROR: Version is required. Provide with --version or set VERSION environment variable" + exit 1 +end + +# Build workflow run URL if in GitHub Actions +if ENV['GITHUB_ACTIONS'] && ENV['GITHUB_SERVER_URL'] && ENV['GITHUB_REPOSITORY'] && ENV['GITHUB_RUN_ID'] + workflow_run_url = "#{ENV['GITHUB_SERVER_URL']}/#{ENV['GITHUB_REPOSITORY']}/actions/runs/#{ENV['GITHUB_RUN_ID']}" +end + +puts "Lighthouse Audit Script" +puts "======================" +puts "Site URL: #{site_url}" +puts "Version: #{version}" +puts "Commit SHA: #{commit_sha}" +puts "Workflow Run URL: #{workflow_run_url || 'N/A'}" +puts "" + +# Check if Lighthouse is installed +lighthouse_path = `which lighthouse`.strip +if lighthouse_path.empty? + puts "ERROR: Lighthouse CLI not found. Install with: npm install -g lighthouse" + exit 1 +end + +puts "Using Lighthouse from: #{lighthouse_path}" + +# Run Lighthouse audit +puts "Running Lighthouse audit against #{site_url}..." + +# Create a temporary output file +output_file = "lighthouse-report-#{Time.now.to_i}.json" + +# Run lighthouse with JSON output +# Flags explained: +# --format=json: Output results as JSON for programmatic parsing +# --output-path=#{output_file}: Save report to specified file +# --chrome-flags='--no-sandbox': Required for running Chrome in containerized CI environments +# --throttling-method=simulate: Use simulated throttling for consistent audit environments +# --quiet: Suppress CLI progress output to keep logs clean +cmd = "lighthouse #{site_url} --format=json --output-path=#{output_file} --chrome-flags='--no-sandbox' --throttling-method=simulate --quiet" +puts "Executing: #{cmd}" + +success = system(cmd) + +unless success + puts "ERROR: Lighthouse execution failed" + exit 1 +end + +unless File.exist?(output_file) + puts "ERROR: Lighthouse report not generated" + exit 1 +end + +# Parse the Lighthouse report +begin + report = JSON.parse(File.read(output_file)) +rescue JSON::ParserError => e + puts "ERROR: Invalid JSON in Lighthouse report: #{e.message}" + exit 1 +end + +# Extract scores (Lighthouse v6+ format) +categories = report.dig('categories') || {} +performance = (categories.dig('performance', 'score') || 0) * 100 +accessibility = (categories.dig('accessibility', 'score') || 0) * 100 +seo = (categories.dig('seo', 'score') || 0) * 100 + +# Round to nearest integer +performance = performance.round +accessibility = accessibility.round +seo = seo.round + +puts "" +puts "Lighthouse Scores:" +puts " Performance: #{performance}" +puts " Accessibility: #{accessibility}" +puts " SEO: #{seo}" +puts "" + +# Clean up temporary report +File.delete(output_file) + +# Load existing quality log +quality_log_path = File.join(__dir__, '..', '_data', 'quality_log.yml') +entries = [] + +if File.exist?(quality_log_path) + begin + # Use YAML.safe_load for security (prevents arbitrary Ruby code execution). + # YAML.safe_load parses the file but ignores comments during parsing. + # This is acceptable because the script regenerates fresh header comments + # on each write to maintain documentation without relying on comment preservation. + content = File.read(quality_log_path) + parsed = YAML.safe_load(content) || [] + entries = parsed.is_a?(Array) ? parsed : [] + rescue YAML::ParseError => e + puts "WARNING: Could not parse existing quality log: #{e.message}" + puts "Starting with empty entries list" + entries = [] + end +end + +# Create new entry +new_entry = { + 'version' => version.to_s, + 'release_date' => Date.today.to_s, + 'commit_sha' => commit_sha, + 'performance' => performance, + 'accessibility' => accessibility, + 'seo' => seo +} + +new_entry['workflow_run_url'] = workflow_run_url if workflow_run_url + +# Append entry to log +entries << new_entry + +# Write back to quality log +File.open(quality_log_path, 'w') do |f| + # Write header comments + f.write("# Quality Log - Lighthouse Audit Metrics\n") + f.write("# Stores performance, accessibility, and SEO scores for each released version.\n") + f.write("# Format: stable YAML with append-only entries, one per released version.\n") + f.write("# Version field format: integer (e.g., 1631).\n") + f.write("# Scores: 0-100 integer scale (100 is best).\n") + f.write("#\n") + f.write("# Structure:\n") + f.write("# - version: \n") + f.write("# release_date: \n") + f.write("# commit_sha: \n") + f.write("# performance: <0-100>\n") + f.write("# accessibility: <0-100>\n") + f.write("# seo: <0-100>\n") + f.write("# workflow_run_url: \n") + f.write("#\n") + f.write("# This file is updated automatically by the lighthouse-audit workflow.\n") + f.write("# Do not edit manually; append entries only via the automated workflow.\n") + f.write("\n") + f.write(YAML.dump(entries)) +end + +puts "Quality log updated: #{quality_log_path}" +puts "Entry: #{new_entry.inspect}" +puts "" +puts "Lighthouse audit complete!"