From 0d8bfcc6abb2eeab42e4ccb887d95372cf35a8e7 Mon Sep 17 00:00:00 2001
From: Daniel Eremeev <danieleremeev@Mini-di-Daniel.lan>
Date: Mon, 12 Oct 2026 15:35:22 +0200
Subject: [PATCH] fix: add Amazonbot and MistralAI-Training to the AI training
 opt-out

---
 docs/content/4.api/1.config.md |  3 ++-
 src/const.ts                   |  6 ++++++
 test/e2e/ai-training.test.ts   |  2 ++
 test/unit/const.test.ts        | 17 ++++++++++++++++-
 4 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/docs/content/4.api/1.config.md b/docs/content/4.api/1.config.md
index c35314f6..9b405e84 100644
--- a/docs/content/4.api/1.config.md
+++ b/docs/content/4.api/1.config.md
@@ -146,7 +146,7 @@ Some user-triggered agents may ignore robots.txt rules.
 
 - `false`: Adds no AI blocking group.
 - `true`: Blocks every token in `AiBots`, including search and user-directed retrieval agents.
-- `'training'`: Blocks `GPTBot`, `ClaudeBot`, `Google-Extended`, `meta-externalagent`, `Applebot-Extended`, and `CCBot`.
+- `'training'`: Blocks `GPTBot`, `ClaudeBot`, `Google-Extended`, `meta-externalagent`, `Amazonbot`, `MistralAI-Training`, `Applebot-Extended`, and `CCBot`.
 
 The `'training'` mode leaves dedicated search and answer agents unblocked, including OpenAI, Claude, and Perplexity agents.
 Other robots.txt groups and site indexing rules still apply.
@@ -154,6 +154,7 @@ The module maintains this curated list locally, with operator sources in [`src/c
 
 [Google-Extended](https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers#google-extended) controls both Gemini training and grounding.
 Blocking it also opts out of grounding in Gemini Apps and Vertex AI, while preserving Google Search inclusion.
+[Amazonbot](https://developer.amazon.com/amazonbot) may be used to train Amazon AI models. Amazon's search and answer agents, Amzn-SearchBot and Amzn-User, stay unblocked.
 [CCBot](https://commoncrawl.org/about) supplies a shared web archive used for training and other purposes.
 Robots.txt cannot guarantee that content stays out of all training datasets.
 
diff --git a/src/const.ts b/src/const.ts
index 5f223685..24e3e8c0 100644
--- a/src/const.ts
+++ b/src/const.ts
@@ -74,6 +74,7 @@ export const AiBots = [
   'ImagesiftBot',
   'MistralAI-User',
   'MistralAI-Index',
+  'MistralAI-Training',
   'AI2Bot',
   'DuckAssistBot',
   'FirecrawlAgent',
@@ -96,6 +97,11 @@ export const AiTrainingBots = [
   'Google-Extended',
   // https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/
   'meta-externalagent',
+  // https://developer.amazon.com/amazonbot
+  // Amazon search and answers use Amzn-SearchBot and Amzn-User, which Amazon says are not used for training.
+  'Amazonbot',
+  // https://docs.mistral.ai/robots
+  'MistralAI-Training',
   // https://support.apple.com/en-us/119829
   'Applebot-Extended',
   // https://commoncrawl.org/about
diff --git a/test/e2e/ai-training.test.ts b/test/e2e/ai-training.test.ts
index c2a5faf7..f5943baa 100644
--- a/test/e2e/ai-training.test.ts
+++ b/test/e2e/ai-training.test.ts
@@ -27,6 +27,8 @@ describe('training opt-out', () => {
       'Google-Extended',
       'Applebot-Extended',
       'meta-externalagent',
+      'Amazonbot',
+      'MistralAI-Training',
       'CCBot',
     ]))
     for (const bot of [
diff --git a/test/unit/const.test.ts b/test/unit/const.test.ts
index 3ecb3621..42d11df9 100644
--- a/test/unit/const.test.ts
+++ b/test/unit/const.test.ts
@@ -1,5 +1,5 @@
 import { describe, expect, it } from 'vitest'
-import { AiBots } from '../../src/const'
+import { AiBots, AiTrainingBots } from '../../src/const'
 
 describe('aiBots', () => {
   it('contains unique user-agent tokens', () => {
@@ -21,3 +21,18 @@ describe('aiBots', () => {
     expect(AiBots).not.toContain('GoogleOther')
   })
 })
+
+describe('aiTrainingBots', () => {
+  it('is a subset of AiBots, so `true` blocks everything `training` blocks', () => {
+    expect(AiBots).toEqual(expect.arrayContaining(AiTrainingBots))
+  })
+
+  it('includes the documented training crawlers', () => {
+    expect(AiTrainingBots).toEqual(expect.arrayContaining(['Amazonbot', 'MistralAI-Training']))
+  })
+
+  it('leaves dedicated search and user-directed agents out', () => {
+    for (const bot of ['Amzn-SearchBot', 'Amzn-User', 'MistralAI-Index', 'MistralAI-User'])
+      expect(AiTrainingBots).not.toContain(bot)
+  })
+})