Audit hardening: evidence, tests, browser checks, recipe verification

.github/pull_request_template.md

@@ -11,3 +11,5 @@
 - [ ] Descriptions are original, factual and free of marketing claims
 - [ ] Recipe changes: test status updated honestly (`tested` only with a passing smoke test)
 - [ ] Added an entry under *Unreleased* in `CHANGELOG.md`
+- [ ] Facts come from upstream sources (LICENSE file, repo, release); no unverified claims
+- [ ] `python -m pytest -q tests` passes; browser tests pass if `docs/catalog.md` or `docs/javascripts/` changed

.github/workflows/ci.yml

@@ -15,6 +15,8 @@ jobs:
       - run: pip install -r requirements.txt
       - name: Schema, duplicates, licences, evidence, no submodules
         run: python scripts/validate.py
+      - name: Unit tests (collector, validator, build)
+        run: python -m pytest -q tests
       - name: Generated files match catalog (and generation is deterministic)
         run: |
           python scripts/build.py && git diff --exit-code
@@ -33,6 +35,30 @@ jobs:
         with:
           args: --offline --no-progress --index-files index.html --root-dir ${{ github.workspace }}/_check --remap 'https://codesbyfebin.github.io/Open-Source-Ai file://${{ github.workspace }}/_check/Open-Source-Ai' --exclude-path _check/Open-Source-Ai/404.html '_check/**/*.html'
           fail: true
+  browser:
+    name: Browser tests (Playwright, desktop + iPhone 13 emulation)
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with: {python-version: '3.12'}
+      - uses: actions/setup-node@v4
+        with: {node-version: '20'}
+      - run: pip install -r requirements.txt
+      - run: mkdocs build --strict -d _site/Open-Source-Ai
+      - working-directory: tests/browser
+        run: npm ci && npx playwright install --with-deps chromium webkit
+      - working-directory: tests/browser
+        run: npx playwright test
+      - if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: playwright-results
+          if-no-files-found: ignore
+          path: |
+            tests/browser/results.json
+            tests/browser/test-results
   upstream-links:
     name: Upstream links (non-blocking)
     runs-on: ubuntu-latest
@@ -54,5 +80,3 @@ jobs:
           docker compose -f recipes/local-chat/docker-compose.yml config --quiet
           docker compose -f recipes/agent-memory-eval/docker-compose.yml config --quiet
       - run: ./recipes/local-chat/smoke.sh
-      - if: always()
-        run: docker compose -f recipes/local-chat/docker-compose.yml down -v

.github/workflows/recipe-agent-memory.yml

@@ -0,0 +1,24 @@
+name: Recipe – agent-memory-eval
+# Heavier end-to-end recipe (~1.3 GB of models). Runs only when the recipe changes, weekly, or on demand.
+# Not a required check.
+on:
+  push:
+    branches: [main]
+    paths: ['recipes/agent-memory-eval/**', '.github/workflows/recipe-agent-memory.yml']
+  pull_request:
+    paths: ['recipes/agent-memory-eval/**', '.github/workflows/recipe-agent-memory.yml']
+  schedule: [{cron: '0 4 * * 1'}]
+  workflow_dispatch:
+permissions: {contents: read}
+jobs:
+  e2e:
+    runs-on: ubuntu-latest
+    timeout-minutes: 40
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with: {python-version: '3.12'}
+      - uses: actions/setup-node@v4
+        with: {node-version: '22'}
+      - run: pip install -r recipes/agent-memory-eval/requirements.txt
+      - run: ./recipes/agent-memory-eval/smoke.sh

.github/workflows/refresh.yml

@@ -3,13 +3,15 @@ on:
   schedule: [{cron: '0 3 * * 1'}]   # weekly, Monday 03:00 UTC
   workflow_dispatch:
 concurrency: {group: refresh, cancel-in-progress: false}
+permissions: {}   # job-level only
 jobs:
   refresh:
     runs-on: ubuntu-latest
     timeout-minutes: 20
     permissions: {contents: write, pull-requests: write}
     steps:
       - uses: actions/checkout@v4
+        with: {persist-credentials: false}
       - uses: actions/setup-python@v5
         with: {python-version: '3.12'}
       - run: pip install -r requirements.txt
@@ -18,21 +20,34 @@ jobs:
           path: .cache/etags.json
           key: etags-${{ github.run_id }}
           restore-keys: etags-
-      - name: Collect evidence from GitHub API
-        env: {GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'}
+      - name: Collect evidence from GitHub API (read-only token)
+        id: collect
+        env: {GITHUB_TOKEN: '${{ github.token }}'}
         run: python scripts/collect.py
-      - name: Regenerate docs and validate
-        run: python scripts/build.py && python scripts/validate.py
+      - name: Regenerate docs, test and validate
+        run: python scripts/build.py && python -m pytest -q tests && python scripts/validate.py
+      # Optional GitHub App token: PRs opened with it trigger CI automatically.
+      # Configure repo variable OSA_APP_ID and secret OSA_APP_PRIVATE_KEY (see CONTRIBUTING.md).
+      - name: Mint GitHub App token (optional)
+        id: app
+        if: ${{ vars.OSA_APP_ID != '' }}
+        uses: actions/create-github-app-token@v2
+        with:
+          app-id: ${{ vars.OSA_APP_ID }}
+          private-key: ${{ secrets.OSA_APP_PRIVATE_KEY }}
       - uses: peter-evans/create-pull-request@v7
         with:
+          token: ${{ steps.app.outputs.token || github.token }}
           branch: bot/evidence-refresh
+          base: main
           delete-branch: true
           commit-message: "catalog: refresh evidence"
-          title: "catalog: weekly evidence refresh"
+          title: "catalog: weekly evidence refresh${{ steps.collect.outputs.failed != '0' && format(' ({0} lookups failed)', steps.collect.outputs.failed) || '' }}"
           body-path: .cache/summary.md
           labels: evidence-refresh
           add-paths: |
             catalog/evidence
             docs
             README.md
             CREDITS.md
+            mkdocs.yml

.gitignore

@@ -3,3 +3,4 @@ site/
 __pycache__/
 .venv/
 _check/
+_site/

CHANGELOG.md

@@ -4,6 +4,13 @@ Catalog changes are recorded here. Evidence refreshes are listed by their pull r
 
 ## Unreleased
 
+- Licences: 15 projects now `needs-review`, each with a recorded `review_reason` (adds Phoenix, Trylon Gateway, llamafile and three AgentDomains repos: source-available or undetected). OpenMemory and PR-Agent re-verified as MIT from their LICENSE files.
+- Collector: failed / rate-limited lookups are reported as failed and keep previous evidence; records `moved` and `fetch_status`. Validator: moved repo and GitHub licence mismatch are errors; zone-aware date checks.
+- Unit tests (collector, validator, build) and Playwright browser tests (Chromium + WebKit, desktop and iPhone 13 emulation, axe-core).
+- Search page: URL state (reload/back/forward), clear button, no-results message, archived badge, focus styles.
+- Recipes: pinned images; local-chat smoke test now proves the reply came from the model; agent-memory-eval end-to-end CI workflow.
+- Refresh workflow: optional GitHub App token, minimal permissions.
+
 - Merged first evidence refresh (PR #1): 70 projects collected, 2 archived upstream (Flowise, Text Generation Inference).
 - Corrected licences from upstream LICENSE files: OpenMemory Apache-2.0 → MIT, PR-Agent AGPL-3.0 → MIT.
 

CONTRIBUTING.md

@@ -32,3 +32,47 @@ Upstream licences are authoritative. Use an SPDX id with `state: declared` only
 ## Evidence refresh
 
 `.github/workflows/refresh.yml` runs weekly (Monday 03:00 UTC) and on demand. It only updates `catalog/evidence/` and regenerated pages, and always opens a pull request for review – it never pushes to `main`. Pull requests opened by the bot do not trigger CI automatically; a maintainer should run `python scripts/validate.py` on the branch (or re-run CI) before merging.
+
+## Verifying facts
+
+- **Licence:** open the upstream LICENSE file (not just GitHub's badge). Use `state: declared` only for a single standard SPDX licence; otherwise `state: needs-review` with a `review_reason` explaining why (custom terms, multiple licences, source-available, undetected).
+- **Repository state:** comes only from `catalog/evidence/`. If validation reports `repo moved`, update `repo:` in `projects.yaml` to the canonical URL.
+- **Archived projects** stay in the catalog and are labelled; don't delete them without a CHANGELOG entry.
+- **Recipes:** `tested` only when a smoke test passes in CI; say what was tested, versions and environment.
+
+## Tests
+
+```bash
+python -m pytest -q tests                      # unit tests
+mkdocs build --strict -d _site/Open-Source-Ai
+cd tests/browser && npm ci && npx playwright install chromium webkit && npx playwright test
+./recipes/local-chat/smoke.sh                  # needs Docker
+```
+
+## Maintainer one-time setup (not done automatically)
+
+These settings are documented here and must be applied by a repository admin.
+
+### Branch protection for `main`
+
+Settings → Branches → Add rule for `main`:
+
+- Require a pull request before merging (1 approval)
+- Require status checks: `Catalog integrity`, `Docs build + internal links`, `Browser tests (Playwright, desktop + iPhone 13 emulation)`, `Recipe smoke test – local-chat`
+- Do **not** require `Upstream links (non-blocking)` or `Recipe – agent-memory-eval` (network-dependent / heavier)
+- Do not allow bypass for the bot
+
+### Evidence refresh PRs and CI
+
+PRs created with the default `GITHUB_TOKEN` **do not trigger** other workflows (a GitHub safety rule), so CI will not start on `bot/evidence-refresh` by itself.
+
+**Option A – GitHub App (recommended):**
+
+1. Settings → Developer settings → GitHub Apps → New. Permissions: *Contents: read & write*, *Pull requests: read & write*. No webhook.
+2. Install it on this repository only.
+3. Add repository variable `OSA_APP_ID` (the App ID) and secret `OSA_APP_PRIVATE_KEY` (the generated private key).
+4. The refresh workflow then uses `actions/create-github-app-token` and CI starts automatically on its PRs.
+
+**Option B – manual fallback (no setup):** open the PR, then either close and reopen it, or run *CI* via *Actions → CI → Run workflow* on the `bot/evidence-refresh` branch. Merge only when blocking checks are green.
+
+The refresh job never pushes to `main`, uses no `pull_request_target`, and the collection step runs before any write token is minted.

CREDITS.md

@@ -8,7 +8,7 @@ Open Source Ai's own content is MIT-licensed © CodesbyFebin. Catalogued project
 | FreeLLMAPI | https://github.com/tashfeenahmed/freellmapi | MIT | declared | 2026-10-20 |
 | OpenCode Free Router | https://github.com/0xgetz/opencode-free-9router | MIT | declared | 2026-10-20 |
 | Bifrost | https://github.com/maximhq/bifrost | Apache-2.0 | declared | 2026-10-20 |
-| Trylon Gateway | https://github.com/trylonai/gateway | BUSL-1.1 | declared | 2026-10-20 |
+| Trylon Gateway | https://github.com/trylonai/gateway | BUSL-1.1 | needs-review | 2026-10-20 |
 | LiteLLM | https://github.com/BerriAI/litellm | MIT (enterprise/ dir separate) | needs-review | 2026-10-20 |
 | Portkey Gateway | https://github.com/Portkey-AI/gateway | MIT | declared | 2026-10-20 |
 | Open WebUI | https://github.com/open-webui/open-webui | Open WebUI License (BSD-3 + branding clause) | needs-review | 2026-10-20 |
@@ -18,15 +18,15 @@ Open Source Ai's own content is MIT-licensed © CodesbyFebin. Catalogued project
 | GPT4All | https://github.com/nomic-ai/gpt4all | MIT | declared | 2026-10-20 |
 | Ollama | https://github.com/ollama/ollama | MIT | declared | 2026-10-20 |
 | llama.cpp | https://github.com/ggml-org/llama.cpp | MIT | declared | 2026-10-20 |
-| llamafile | https://github.com/mozilla-ai/llamafile | Apache-2.0 | declared | 2026-10-20 |
+| llamafile | https://github.com/mozilla-ai/llamafile | Apache-2.0 | needs-review | 2026-10-20 |
 | LocalAI | https://github.com/mudler/LocalAI | MIT | declared | 2026-10-20 |
 | Text Generation WebUI | https://github.com/oobabooga/textgen | AGPL-3.0 | declared | 2026-10-20 |
 | vLLM | https://github.com/vllm-project/vllm | Apache-2.0 | declared | 2026-10-20 |
 | SGLang | https://github.com/sgl-project/sglang | Apache-2.0 | declared | 2026-10-20 |
 | Text Generation Inference | https://github.com/huggingface/text-generation-inference | Apache-2.0 | declared | 2026-10-20 |
 | Triton Inference Server | https://github.com/triton-inference-server/server | BSD-3-Clause | declared | 2026-10-20 |
 | Mem0 | https://github.com/mem0ai/mem0 | Apache-2.0 | declared | 2026-10-20 |
-| OpenMemory | https://github.com/mem0ai/openmemory | MIT | declared | 2026-10-20 |
+| OpenMemory | https://github.com/mem0ai/openmemory | MIT | declared | 2026-10-21 |
 | Letta | https://github.com/letta-ai/letta | Apache-2.0 | declared | 2026-10-20 |
 | Graphiti | https://github.com/getzep/graphiti | Apache-2.0 | declared | 2026-10-20 |
 | LangChain | https://github.com/langchain-ai/langchain | MIT | declared | 2026-10-20 |
@@ -42,7 +42,7 @@ Open Source Ai's own content is MIT-licensed © CodesbyFebin. Catalogued project
 | OpenCode | https://github.com/anomalyco/opencode | MIT | declared | 2026-10-20 |
 | Cline | https://github.com/cline/cline | Apache-2.0 | declared | 2026-10-20 |
 | Open Code Review | https://github.com/alibaba/open-code-review | Apache-2.0 | declared | 2026-10-20 |
-| PR-Agent | https://github.com/The-PR-Agent/pr-agent | MIT | declared | 2026-10-20 |
+| PR-Agent | https://github.com/The-PR-Agent/pr-agent | MIT | declared | 2026-10-21 |
 | Dify | https://github.com/langgenius/dify | Dify Open Source License (Apache-2.0 + conditions) | needs-review | 2026-10-20 |
 | Flowise | https://github.com/FlowiseAI/Flowise | Apache-2.0 (enterprise dir separate) | needs-review | 2026-10-20 |
 | Langflow | https://github.com/langflow-ai/langflow | MIT | declared | 2026-10-20 |
@@ -55,7 +55,7 @@ Open Source Ai's own content is MIT-licensed © CodesbyFebin. Catalogued project
 | Milvus | https://github.com/milvus-io/milvus | Apache-2.0 | declared | 2026-10-20 |
 | Weaviate | https://github.com/weaviate/weaviate | BSD-3-Clause (core) | needs-review | 2026-10-20 |
 | Langfuse | https://github.com/langfuse/langfuse | MIT (ee/ dirs separate) | needs-review | 2026-10-20 |
-| Phoenix | https://github.com/Arize-ai/phoenix | Elastic-2.0 | declared | 2026-10-20 |
+| Phoenix | https://github.com/Arize-ai/phoenix | Elastic-2.0 | needs-review | 2026-10-20 |
 | promptfoo | https://github.com/promptfoo/promptfoo | MIT | declared | 2026-10-20 |
 | DeepEval | https://github.com/confident-ai/deepeval | Apache-2.0 | declared | 2026-10-20 |
 | Butterbase | https://github.com/butterbase-ai/butterbase | Apache-2.0 | declared | 2026-10-20 |
@@ -65,9 +65,9 @@ Open Source Ai's own content is MIT-licensed © CodesbyFebin. Catalogued project
 | One Person Company | https://github.com/tashfeenahmed/OnePersonCompany | MIT | declared | 2026-10-20 |
 | Kinetic | https://github.com/tashfeenahmed/kinetic | MIT | declared | 2026-10-20 |
 | CircleChat | https://github.com/tashfeenahmed/circlechat | MIT | declared | 2026-10-20 |
-| AgentDomains | https://github.com/tashfeenahmed/AgentDomains | FSL-1.1-Apache-2.0 | declared | 2026-10-20 |
-| AgentDomains MCP | https://github.com/tashfeenahmed/AgentDomains-mcp | FSL-1.1-Apache-2.0 | declared | 2026-10-20 |
-| AgentDomains Skill | https://github.com/tashfeenahmed/AgentDomains-skill | FSL-1.1-Apache-2.0 | declared | 2026-10-20 |
+| AgentDomains | https://github.com/tashfeenahmed/AgentDomains | FSL-1.1-Apache-2.0 | needs-review | 2026-10-20 |
+| AgentDomains MCP | https://github.com/tashfeenahmed/AgentDomains-mcp | FSL-1.1-Apache-2.0 | needs-review | 2026-10-20 |
+| AgentDomains Skill | https://github.com/tashfeenahmed/AgentDomains-skill | FSL-1.1-Apache-2.0 | needs-review | 2026-10-20 |
 | ScallopBot | https://github.com/tashfeenahmed/scallopbot | MIT | declared | 2026-10-20 |
 | Cashmaxx | https://github.com/tashfeenahmed/cashmaxx | MIT | declared | 2026-10-20 |
 | Jev Deep Research | https://github.com/tashfeenahmed/jev-deepresearch | MIT | declared | 2026-10-20 |

catalog/evidence/agentdomains-mcp.json

@@ -1,6 +1,10 @@
 {
   "canonical_repo": "https://github.com/tashfeenahmed/AgentDomains-mcp",
   "default_branch": "main",
+  "fetch_status": {
+    "at": "2026-10-20T19:54:06Z",
+    "result": "ok"
+  },
   "id": "agentdomains-mcp",
   "last_commit": "2026-09-30T13:17:04Z",
   "latest_release": null,
@@ -9,8 +13,9 @@
     "source": "https://api.github.com/repos/tashfeenahmed/AgentDomains-mcp/license",
     "state": "needs-review"
   },
+  "moved": false,
   "pushed_at": "2026-09-30T13:17:04Z",
-  "retrieved_at": "2026-10-20T18:52:53Z",
+  "retrieved_at": "2026-10-20T19:54:06Z",
   "source": "https://api.github.com/repos/tashfeenahmed/AgentDomains-mcp",
   "stars": 2,
   "state": "active"

catalog/evidence/agentdomains-skill.json

@@ -1,6 +1,10 @@
 {
   "canonical_repo": "https://github.com/tashfeenahmed/AgentDomains-skill",
   "default_branch": "main",
+  "fetch_status": {
+    "at": "2026-10-20T19:54:06Z",
+    "result": "ok"
+  },
   "id": "agentdomains-skill",
   "last_commit": "2026-10-17T20:25:20Z",
   "latest_release": null,
@@ -9,8 +13,9 @@
     "source": "https://api.github.com/repos/tashfeenahmed/AgentDomains-skill/license",
     "state": "needs-review"
   },
+  "moved": false,
   "pushed_at": "2026-10-17T20:25:23Z",
-  "retrieved_at": "2026-10-20T18:52:53Z",
+  "retrieved_at": "2026-10-20T19:54:06Z",
   "source": "https://api.github.com/repos/tashfeenahmed/AgentDomains-skill",
   "stars": 10,
   "state": "active"

catalog/evidence/agentdomains.json

@@ -1,6 +1,10 @@
 {
   "canonical_repo": "https://github.com/tashfeenahmed/AgentDomains",
   "default_branch": "main",
+  "fetch_status": {
+    "at": "2026-10-20T19:54:06Z",
+    "result": "ok"
+  },
   "id": "agentdomains",
   "last_commit": "2026-10-17T19:16:41Z",
   "latest_release": {
@@ -12,8 +16,9 @@
     "source": "https://api.github.com/repos/tashfeenahmed/AgentDomains/license",
     "state": "needs-review"
   },
+  "moved": false,
   "pushed_at": "2026-10-17T20:24:07Z",
-  "retrieved_at": "2026-10-20T18:52:53Z",
+  "retrieved_at": "2026-10-20T19:54:06Z",
   "source": "https://api.github.com/repos/tashfeenahmed/AgentDomains",
   "stars": 39,
   "state": "active"

catalog/evidence/agno.json

@@ -1,6 +1,10 @@
 {
   "canonical_repo": "https://github.com/agno-agi/agno",
   "default_branch": "main",
+  "fetch_status": {
+    "at": "2026-10-20T19:54:06Z",
+    "result": "ok"
+  },
   "id": "agno",
   "last_commit": "2026-10-20T16:30:55Z",
   "latest_release": {
@@ -12,8 +16,9 @@
     "source": "https://api.github.com/repos/agno-agi/agno/license",
     "state": "detected"
   },
+  "moved": false,
   "pushed_at": "2026-10-20T16:31:36Z",
-  "retrieved_at": "2026-10-20T18:52:53Z",
+  "retrieved_at": "2026-10-20T19:54:06Z",
   "source": "https://api.github.com/repos/agno-agi/agno",
   "stars": 42836,
   "state": "active"

catalog/evidence/aider.json

@@ -1,6 +1,10 @@
 {
   "canonical_repo": "https://github.com/Aider-AI/aider",
   "default_branch": "main",
+  "fetch_status": {
+    "at": "2026-10-20T19:54:06Z",
+    "result": "ok"
+  },
   "id": "aider",
   "last_commit": "2026-05-22T14:02:20Z",
   "latest_release": {
@@ -12,8 +16,9 @@
     "source": "https://api.github.com/repos/Aider-AI/aider/license",
     "state": "detected"
   },
+  "moved": false,
   "pushed_at": "2026-05-22T14:02:20Z",
-  "retrieved_at": "2026-10-20T18:52:53Z",
+  "retrieved_at": "2026-10-20T19:54:06Z",
   "source": "https://api.github.com/repos/Aider-AI/aider",
   "stars": 49663,
   "state": "active"

catalog/evidence/autogen.json

@@ -1,6 +1,10 @@
 {
   "canonical_repo": "https://github.com/microsoft/autogen",
   "default_branch": "main",
+  "fetch_status": {
+    "at": "2026-10-20T19:54:06Z",
+    "result": "ok"
+  },
   "id": "autogen",
   "last_commit": "2026-10-16T16:14:36Z",
   "latest_release": {
@@ -12,8 +16,9 @@
     "source": "https://api.github.com/repos/microsoft/autogen/license",
     "state": "detected"
   },
+  "moved": false,
   "pushed_at": "2026-10-16T16:14:37Z",
-  "retrieved_at": "2026-10-20T18:52:53Z",
+  "retrieved_at": "2026-10-20T19:54:06Z",
   "source": "https://api.github.com/repos/microsoft/autogen",
   "stars": 61497,
   "state": "active"