feat(content-signal): publish HTTP policy headers

.gitignore

@@ -57,3 +57,4 @@ Temporary Items
 
 # PR Lens writes its previews here. They are rebuilt on demand.
 .pr-lens/
+.wrangler/

docs/content/2.guides/9.ai-directives.md

@@ -173,6 +173,30 @@ export default defineNuxtConfig({
 })
 ```
 
+### HTTP headers
+
+With `robots.header: true`{lang="ts"}, the module also sends a site-wide `Content-Signal`{lang="txt"} response header.
+It derives this header from consistent `User-agent: *`{lang="txt"} declarations, including a merged robots file.
+Missing permissions remain unspecified.
+An explicit response header takes precedence.
+
+The module omits the derived header when any group targets a specific user-agent.
+Path exceptions and conflicting permissions also prevent a derived header.
+
+Cloudflare allows 100 static header rules.
+If the fallback exceeds this limit, the build fails after all modules finish writing headers.
+The module retains policy rules. Use a shared header rule or reduce other rules.
+A site-wide header cannot represent user-agent or path exceptions.
+Those directives remain in `robots.txt`{lang="txt"}.
+Set explicit route headers when you need HTTP policy for these configurations.
+
+[Cloudflare](https://cloudflare.com) builds also publish the fallback in `_headers`{lang="txt"} for prerendered assets.
+Explicit static policies and header removals take precedence.
+Other static hosts need their own header configuration.
+
+[Cloudflare Markdown for Agents](https://developers.cloudflare.com/fundamentals/reference/markdown-for-agents/#content-signals-policy) preserves an origin policy header.
+Without one, the converter adds a policy that allows training, search, and AI input.
+
 ## Vendor-Specific AI Tokens
 
 While `Content-Usage` and `Content-Signal` are emerging standards, some major AI providers offer specific User-Agent tokens to opt-out of AI training while maintaining search visibility.

src/content-signal-headers.ts

@@ -0,0 +1,32 @@
+/** Add the fallback first, so explicit Cloudflare rules can detach it before setting their policy. */
+export function mergeContentSignalHeaders(contents: string, pattern: string, policy: string, limit: number | null = 100): string {
+  const blocks = contents.split(/(?=^[^\s#])/m)
+  // A site's own catch-all policy or removal is authoritative.
+  const isBroad = (block: string) => block.split(/\r?\n/, 1)[0] === pattern
+  const broad = blocks.filter(isBroad)
+  if (broad.some(block => /^\s+(?:content-signal\s*:|!\s*content-signal\s*$)/im.test(block)))
+    return contents
+  const explicit = blocks.filter(block => !isBroad(block)).map((block) => {
+    if (!/^\s+content-signal\s*:/im.test(block) || /^\s+!\s*content-signal\s*$/im.test(block))
+      return block
+    return block.replace(/^(\s+)(content-signal\s*:)/im, '$1! Content-Signal\n$1$2')
+  }).join('')
+  const fallback = broad.length
+    ? broad.map(block => `${block.replace(/^([^\r\n]*)(?:\r?\n|$)/, `$1\n  Content-Signal: ${policy}\n`).trimEnd()}\n\n`).join('')
+    : `${pattern}\n  Content-Signal: ${policy}\n\n`
+  const merged = `${fallback}${explicit}`
+  if (limit !== null)
+    assertContentSignalHeaderBudget(merged, limit)
+  return merged
+}
+
+export function countStaticHeaderRules(contents: string): number {
+  return contents.split(/\r?\n/).filter(line => /^[^\s#]/.test(line)).length
+}
+
+/** Policy rules cannot be dropped to make a deployment fit. */
+export function assertContentSignalHeaderBudget(contents: string, limit = 100): void {
+  const total = countStaticHeaderRules(contents)
+  if (total > limit)
+    throw new Error(`Content-Signal fallback exceeds Cloudflare's ${limit}-rule header limit (${total}). Use a shared header rule or reduce other rules.`)
+}

src/module.ts

@@ -1,6 +1,7 @@
 import type { FileAfterParseHook } from '@nuxt/content'
 import type { NuxtPage } from 'nuxt/schema'
 import type { Arrayable, AutoI18nConfig, NuxtRobotsRuntimeConfig, RobotsGroupInput, RobotsGroupResolved } from './util'
+import { readFileSync } from 'node:fs'
 import fsp, { readFile } from 'node:fs/promises'
 import { relative } from 'node:path'
 import { fileURLToPath, pathToFileURL } from 'node:url'
@@ -23,6 +24,7 @@ import { updateSiteConfig, useSiteConfig } from 'nuxt-site-config/kit'
 import { setupNitroRuntimeCompatibility, setupRuntimeAliases, useModuleLogger } from 'nuxtseo-shared/kit'
 import { withoutTrailingSlash, withTrailingSlash } from 'ufo'
 import { AiBots, AiTrainingBots, NonHelpfulBots } from './const'
+import { assertContentSignalHeaderBudget, countStaticHeaderRules, mergeContentSignalHeaders } from './content-signal-headers'
 import { setupDevToolsUI } from './devtools'
 import { applyRobotsHeaderRouteRules, findRobotsHeader, resolveRobotsHeaderRules } from './header-rules'
 import { mapPathForI18nPages, resolveI18nConfig, splitPathForI18nLocales } from './i18n'
@@ -33,6 +35,7 @@ import {
   normaliseRobotsRouteRule,
   normalizeGroup,
   parseRobotsTxt,
+  resolveContentSignalHeader,
   ROBOT_DIRECTIVE_VALUES,
   robotsDirectivesFromObject,
   validateRobots,
@@ -56,7 +59,7 @@ export interface ModuleOptions {
    */
   enabled: boolean
   /**
-   * Should a `X-Robots-Tag` header be added to the response.
+   * Add `X-Robots-Tag` and an eligible site-wide `Content-Signal` header.
    *
    * @default true
    */
@@ -622,6 +625,7 @@ export default defineNuxtModule<ModuleOptions>({
         groups,
         sitemap: config.sitemap,
         header: config.header,
+        contentSignalHeader: config.header ? resolveContentSignalHeader(groups) : undefined,
         robotsEnabledValue: config.robotsEnabledValue,
         robotsDisabledValue: config.robotsDisabledValue,
         cacheControl: config.cacheControl ?? 'max-age=14400, must-revalidate',
@@ -633,6 +637,45 @@ export default defineNuxtModule<ModuleOptions>({
 
     registerTypeTemplates({ nitroCompatibility })
 
+    nuxt.hook('nitro:build:before', (nitro) => {
+      if (!String(nitro.options.preset).startsWith('cloudflare'))
+        return
+      let budget: { _tag: 'Unchanged' } | { _tag: 'Added', path: string } = { _tag: 'Unchanged' }
+      // Hook observers run after all compiled hooks. AI Ready can trim discovery rules first.
+      nitro.hooks.afterEach((hook) => {
+        if (hook.name === 'compiled' && budget._tag === 'Added')
+          assertContentSignalHeaderBudget(readFileSync(budget.path, 'utf8'))
+      })
+      nitro.hooks.hook('compiled', async () => {
+        const policy = (nitro.options.runtimeConfig['nuxt-robots'] as unknown as NuxtRobotsRuntimeConfig).contentSignalHeader
+        if (!policy)
+          return
+        const base = nitro.options.baseURL === '/' ? '' : withoutTrailingSlash(nitro.options.baseURL || '')
+        // Cloudflare reads headers at the asset root, above an application's base directory.
+        const assetRoot = String(nitro.options.preset).includes('pages')
+          ? nitro.options.output.dir
+          : resolve(nitro.options.output.publicDir, ...base.split('/').filter(Boolean).map(() => '..'))
+        const path = resolve(assetRoot, '_headers')
+        const publicPath = resolve(nitro.options.output.publicDir, '_headers')
+        const readHeaders = (file: string) => readFile(file, 'utf8').catch((error: NodeJS.ErrnoException) => {
+          if (error.code === 'ENOENT')
+            return ''
+          throw error
+        })
+        const original = await readHeaders(path)
+        const nested = publicPath === path ? '' : await readHeaders(publicPath)
+        const contents = [original, nested].filter(Boolean).join('\n\n')
+        const pattern = `${base}/*`
+        const merged = mergeContentSignalHeaders(contents, pattern, policy, null)
+        if (countStaticHeaderRules(contents) <= 100 && countStaticHeaderRules(merged) > 100)
+          budget = { _tag: 'Added', path }
+        if (merged !== original)
+          await fsp.writeFile(path, merged)
+        if (nested)
+          await fsp.rm(publicPath)
+      })
+    })
+
     // only prerender for `nuxi generate`
     const isFirebase = nitroPreset === 'firebase'
     // @ts-expected-error nuxt 3
@@ -657,6 +700,8 @@ export default defineNuxtModule<ModuleOptions>({
       handler: resolve('./runtime/server/middleware/injectContext'),
     })
     addServerPlugin(resolve('./runtime/server/plugins/initContext'))
+    if (!isNitro3)
+      addServerPlugin(resolve('./runtime/server/plugins/content-signal'))
     // Nitro 3 merges middleware headers after rendering. Apply the final rule after that merge.
     if (isNitro3) {
       addServerPlugin(addServerTemplate({
@@ -666,6 +711,9 @@ import { defineNitroPlugin } from '#nuxtseo/nitro'
 import { useRuntimeConfigNuxtRobots } from ${JSON.stringify(resolve('./runtime/server/composables/useRuntimeConfigNuxtRobots'))}
 export default defineNitroPlugin((nitroApp) => {
   nitroApp.hooks.hook('response', (response, event) => {
+    const { contentSignalHeader } = useRuntimeConfigNuxtRobots(event)
+    if (contentSignalHeader && !response.headers.has('content-signal'))
+      response.headers.set('Content-Signal', contentSignalHeader)
     const rule = event.context.robots?.rule
     if (rule && useRuntimeConfigNuxtRobots(event).header)
       response.headers.set('X-Robots-Tag', rule)

src/runtime/server/plugins/content-signal.ts

@@ -0,0 +1,11 @@
+import { getResponseHeader, setHeader } from '#nuxtseo/h3'
+import { defineNitroPlugin } from '#nuxtseo/nitro'
+import { useRuntimeConfigNuxtRobots } from '../composables/useRuntimeConfigNuxtRobots'
+
+export default defineNitroPlugin((nitro) => {
+  nitro.hooks.hook('beforeResponse', (event) => {
+    const { contentSignalHeader } = useRuntimeConfigNuxtRobots(event)
+    if (contentSignalHeader && getResponseHeader(event, 'content-signal') === undefined)
+      setHeader(event, 'Content-Signal', contentSignalHeader)
+  })
+})

src/runtime/types.ts

@@ -212,6 +212,7 @@ export interface NuxtRobotsRuntimeConfig {
   groups: RobotsGroupResolved[]
   sitemap: string[]
   header: boolean
+  contentSignalHeader?: string
   robotsEnabledValue: string
   robotsDisabledValue: string
   cacheControl: string | false

src/util.ts

@@ -20,6 +20,29 @@ import { matchDefaultBotTarget } from './generated-bot-matcher'
 export * from './const-bots'
 export { AiBots, NonHelpfulBots }
 
+/** A response header can express one site-wide policy, without user-agent exceptions. */
+export function resolveContentSignalHeader(groups: RobotsGroupInput[]): string | undefined {
+  const permissions = new Map<string, string>()
+  for (const input of groups) {
+    const group = normalizeGroup(input)
+    if (group.userAgent.length !== 1 || group.userAgent[0] !== '*')
+      return
+    const signals = group.contentSignal || []
+    if (!signals.length)
+      continue
+    for (const line of signals) {
+      for (const entry of line.split(',')) {
+        const pair = entry.trim().match(/^(ai-train|search|ai-input)=(yes|no)$/)
+        if (!pair || (permissions.has(pair[1]!) && permissions.get(pair[1]!) !== pair[2]))
+          return
+        permissions.set(pair[1]!, pair[2]!)
+      }
+    }
+  }
+  if (permissions.size)
+    return [...permissions].map(([name, value]) => `${name}=${value}`).join(', ')
+}
+
 /**
  * Predefined robot directive values map
  */

test/e2e/contentSignalHeaders.test.ts

@@ -0,0 +1,27 @@
+import { createResolver } from '@nuxt/kit'
+import { fetch, setup } from '@nuxt/test-utils'
+import { describe, expect, it } from 'vitest'
+
+const { resolve } = createResolver(import.meta.url)
+
+await setup({
+  rootDir: resolve('../fixtures/basic'),
+  build: true,
+  server: true,
+  nuxtConfig: {
+    robots: { groups: [{ userAgent: '*', contentSignal: 'ai-train=no, search=yes, ai-input=yes' }] },
+    routeRules: { '/about': { headers: { 'CONTENT-SIGNAL': 'search=no' } } },
+  },
+})
+
+describe('content-Signal response headers', () => {
+  it('publishes the wildcard robots policy', async () => {
+    const response = await fetch('/')
+    expect(response.headers.get('content-signal')).toBe('ai-train=no, search=yes, ai-input=yes')
+  })
+
+  it('preserves an explicit route policy in any casing', async () => {
+    const response = await fetch('/about')
+    expect(response.headers.get('content-signal')).toBe('search=no')
+  })
+})

test/e2e/contentSignalHeadersBudget.test.ts

@@ -0,0 +1,16 @@
+import { createResolver } from '@nuxt/kit'
+import { setup } from '@nuxt/test-utils'
+import { describe, expect, it } from 'vitest'
+import { readHeadersFile } from './utils/headers'
+
+const { resolve } = createResolver(import.meta.url)
+
+await setup({ rootDir: resolve('../fixtures/content-signal-budget'), build: true, server: false })
+
+describe('content-Signal header budget', () => {
+  it('allows a later compiler to trim optional rules before checking the final budget', () => {
+    const contents = readHeadersFile()
+    expect(contents.split('\n').filter(line => /^[^\s#]/.test(line))).toHaveLength(100)
+    expect(contents).toContain('Content-Signal: ai-train=no')
+  })
+})

test/e2e/contentSignalHeadersScoped.test.ts

@@ -0,0 +1,30 @@
+import { createResolver } from '@nuxt/kit'
+import { $fetch, fetch, setup } from '@nuxt/test-utils'
+import { describe, expect, it } from 'vitest'
+
+const { resolve } = createResolver(import.meta.url)
+
+await setup({
+  rootDir: resolve('../fixtures/basic'),
+  build: true,
+  server: true,
+  nuxtConfig: {
+    robots: {
+      groups: [
+        { userAgent: '*', contentSignal: 'ai-train=no' },
+        { userAgent: 'GPTBot', contentSignal: 'ai-train=yes' },
+      ],
+    },
+  },
+})
+
+describe('user-agent Content Signal policies', () => {
+  it('keeps scoped policies in robots.txt without flattening them into HTTP', async () => {
+    const response = await fetch('/')
+    const robots = await $fetch<string>('/robots.txt')
+
+    expect(response.headers.get('content-signal')).toBeNull()
+    expect(robots).toMatch(/User-agent: \*\nContent-Signal: ai-train=no/)
+    expect(robots).toMatch(/User-agent: GPTBot\nContent-Signal: ai-train=yes/)
+  })
+})

test/e2e/contentSignalHeadersStatic.test.ts

@@ -0,0 +1,49 @@
+import { createResolver } from '@nuxt/kit'
+import { setup } from '@nuxt/test-utils'
+import { describe, expect, it } from 'vitest'
+import { readHeadersFile } from './utils/headers'
+
+const { resolve } = createResolver(import.meta.url)
+
+await setup({
+  rootDir: resolve('../fixtures/content-signal-headers'),
+  build: true,
+  server: false,
+})
+
+function readPolicy(path: string) {
+  const headers = new Headers()
+  let matches = false
+  for (const line of readHeadersFile().split('\n')) {
+    if (!line.trim())
+      continue
+    if (!/^\s/.test(line)) {
+      matches = new RegExp(`^${line.trim().replace(/[.+?^${}()|[\]\\]/g, '\\$&').replaceAll('*', '.*')}$`).test(path)
+      continue
+    }
+    if (!matches)
+      continue
+    if (/^\s*!\s*content-signal\s*$/i.test(line))
+      headers.delete('content-signal')
+    const value = line.trim().match(/^content-signal:(.*)$/i)
+    if (value)
+      headers.append('content-signal', value[1]!.trim())
+  }
+  return headers.get('content-signal')
+}
+
+describe('cloudflare static Content-Signal headers', () => {
+  it('serves the wildcard policy without calling the server', () => {
+    expect(readPolicy('/docs/')).toBe('ai-train=no, search=yes')
+    expect(readPolicy('/docs/policy.md')).toBe('ai-train=no, search=yes')
+    expect(readPolicy('/elsewhere')).toBeNull()
+  })
+
+  it('keeps the explicit static policy without joining the default', () => {
+    expect(readPolicy('/docs/api/data.json')).toBe('search=no')
+  })
+
+  it('preserves an explicit header removal', () => {
+    expect(readPolicy('/docs/removed/page')).toBeNull()
+  })
+})

test/e2e/noHeader.test.ts

@@ -12,6 +12,7 @@ await setup({
     robots: {
       debug: true,
       header: false,
+      groups: [{ userAgent: '*', contentSignal: 'ai-train=no' }],
     },
   },
 })
@@ -20,5 +21,6 @@ describe('noHeader', () => {
   it('basic', async () => {
     const { headers, _data } = await $fetch.raw('/')
     expect(headers.get('X-Robots-Tag')).toBe(null)
+    expect(headers.get('Content-Signal')).toBe(null)
   })
 })

test/fixtures/content-signal-budget/nuxt.config.ts

@@ -0,0 +1,31 @@
+import { readFile, writeFile } from 'node:fs/promises'
+import { resolve } from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { useNuxt } from '@nuxt/kit'
+
+export default defineNuxtConfig({
+  extends: ['../static-headers'],
+  alias: { '@nuxtjs/robots/util': fileURLToPath(new URL('../../../src/util.ts', import.meta.url)) },
+  robots: { robotsTxt: false, groups: [{ userAgent: '*', contentSignal: 'ai-train=no' }] },
+  nitro: { preset: 'cloudflare-module', cloudflare: { deployConfig: true } },
+  hooks: {
+    'nitro:build:before': (nitro) => {
+      nitro.hooks.hook('compiled', () => writeFile(resolve(nitro.options.output.publicDir, '_headers'),
+        Array.from({ length: 100 }, (_, index) => `/page-${index}\n  X-Page: ${index}\n`).join('\n')))
+    },
+    'modules:done': () => {
+      if (process.env.NUXT_TEST_SKIP_HEADER_TRIM === 'true')
+        return
+      const nuxt = useNuxt()
+      nuxt.hook('nitro:build:before', (nitro) => {
+        nitro.hooks.hook('compiled', async () => {
+          const path = resolve(nitro.options.output.publicDir, '_headers')
+          const headers = await readFile(path, 'utf8')
+          if (!headers.includes('Content-Signal: ai-train=no'))
+            throw new Error('The later compiler must observe the generated policy.')
+          await writeFile(path, headers.replace('/page-99\n  X-Page: 99\n', ''))
+        })
+      })
+    },
+  },
+})