feat: new Content-Usage directive (#226)

Author: harlan-zw

docs/content/2.guides/1.robots-txt.md

@@ -60,9 +60,24 @@ The following rules are parsed from your `robots.txt` file:
 - `Disallow` - An array of paths to disallow for the user-agent.
 - `Allow` - An array of paths to allow for the user-agent.
 - `Sitemap` - An array of sitemap URLs to include in the generated sitemap.
+- `Content-Usage` - Content Signals for expressing AI usage preferences (see [Content Signals](#content-signals) below).
 
 This parsed data will be shown for environments that are `indexable`.
 
+## Content Signals
+
+Content Signals allow you to express preferences about how AI systems should interact with your content using the `Content-Usage` directive.
+
+```txt [robots.txt]
+User-agent: *
+Allow: /
+Content-Usage: ai=n
+Content-Usage: /public/ train-ai=y
+Content-Usage: /restricted/ ai=n train-ai=n
+```
+
+See the emerging [IETF AI Preferences specification](https://datatracker.ietf.org/doc/draft-ietf-aipref-attach/) for more details.
+
 ## Conflicting `public/robots.txt`
 
 To ensure other modules can integrate with your generated robots file, you must not have a `robots.txt` file in your `public` folder.

docs/content/2.guides/3.nuxt-config.md

@@ -65,3 +65,44 @@ User-agent: AdsBot-Google-Mobile-Apps
 Disallow: /admin
 Allow: /admin/login
 ```
+
+## Content Signals Configuration
+
+You can configure Content Signals (AI usage preferences) programmatically using the `contentUsage` option in your groups:
+
+```ts [nuxt.config.ts]
+export default defineNuxtConfig({
+  robots: {
+    groups: [
+      {
+        userAgent: '*',
+        allow: '/',
+        contentUsage: [
+          'ai=n', // Disable AI usage globally
+          '/docs/ train-ai=y', // Allow AI training on docs
+          '/api/ ai=n train-ai=n' // Disable all AI usage for API
+        ]
+      }
+    ]
+  }
+})
+```
+
+This will generate:
+
+```robots-txt [robots.txt]
+User-agent: *
+Allow: /
+Content-Usage: ai=n
+Content-Usage: /docs/ train-ai=y
+Content-Usage: /api/ ai=n train-ai=n
+```
+
+### Content-Usage Options
+
+The `contentUsage` field accepts an array of strings with the following formats:
+
+- **Global preferences**: `'ai=n'`, `'train-ai=y'`
+- **Path-specific preferences**: `'/path/ ai=n'`, `'/docs/ train-ai=y'`
+
+See the [Content Signals guide](/docs/robots/guides/robots-txt#content-signals) for more detailed information about Content Signals and supported AI preferences.

docs/content/3.api/1.config.md

@@ -41,13 +41,24 @@ export default defineNuxtConfig({
         userAgent: ['AdsBot-Google-Mobile', 'AdsBot-Google-Mobile-Apps'],
         disallow: ['/admin'],
         allow: ['/admin/login'],
+        contentUsage: ['ai=n', '/docs/ train-ai=y'],
         comments: 'Allow Google AdsBot to index the login page but no-admin pages'
       },
     ]
   }
 })
 ```
 
+### Group Configuration Options
+
+Each group object supports the following properties:
+
+- `userAgent?: string | string[]` - The user agent(s) to apply rules to. Defaults to `['*']`
+- `disallow?: string | string[]` - Paths to disallow for the user agent(s)
+- `allow?: string | string[]` - Paths to allow for the user agent(s)
+- `contentUsage?: string | string[]` - Content Signals for AI usage preferences (see [Content Signals guide](/docs/robots/guides/robots-txt#content-signals))
+- `comment?: string | string[]` - Comments to include in the robots.txt file
+
 ## `sitemap: MaybeArray<string>`{lang="ts"}
 
 - Default: `[]`{lang="ts"}

src/runtime/types.ts

@@ -55,6 +55,7 @@ export interface GoogleInput {
   disallow?: Arrayable<string>
   allow?: Arrayable<string>
   userAgent?: Arrayable<string>
+  contentUsage?: Arrayable<string>
   // nuxt-simple-robots internals
   _skipI18n?: boolean
 }
@@ -71,6 +72,8 @@ export interface RobotsGroupResolved {
   host?: string
   // yandex only
   cleanParam?: string[]
+  // content signals / AI preferences
+  contentUsage?: string[]
   // nuxt-simple-robots internals
   _skipI18n?: boolean
   // runtime optimization

src/util.ts

@@ -45,6 +45,7 @@ export function parseRobotsTxt(s: string): ParsedRobotsTxt {
     disallow: [],
     allow: [],
     userAgent: [],
+    contentUsage: [],
   }
   let ln = -1
   // read the contents
@@ -73,6 +74,7 @@ export function parseRobotsTxt(s: string): ParsedRobotsTxt {
             disallow: [],
             allow: [],
             userAgent: [],
+            contentUsage: [],
           }
           createNewGroup = false
         }
@@ -107,6 +109,10 @@ export function parseRobotsTxt(s: string): ParsedRobotsTxt {
           errors.push(`L${ln}: Clean-param directive is only when targeting Yandex user agent.`)
         }
         break
+      case 'content-usage':
+        currentGroup.contentUsage = currentGroup.contentUsage || []
+        currentGroup.contentUsage.push(val)
+        break
       default:
         errors.push(`L${ln}: Unknown directive ${rule} `)
         break
@@ -137,6 +143,39 @@ function validateGroupRules(group: ParsedRobotsTxt['groups'][number], errors: st
       return true
     })
   })
+
+  // Validate Content-Usage directives
+  if (group.contentUsage) {
+    group.contentUsage.forEach((rule) => {
+      if (rule === '') {
+        errors.push(`Content-Usage rule cannot be empty.`)
+        return
+      }
+
+      // Basic validation for Content-Usage format
+      // Format can be: "preference" or "/path preference"
+      const parts = rule.trim().split(/\s+/)
+
+      if (parts.length === 1) {
+        // Global preference like "ai=n" or "train-ai=n"
+        if (!parts[0]?.includes('=')) {
+          errors.push(`Content-Usage rule "${rule}" must contain a preference assignment (e.g., "ai=n").`)
+        }
+      }
+      else if (parts.length >= 2) {
+        // Path-specific preference like "/path ai=n"
+        const path = parts[0]
+        const preference = parts.slice(1).join(' ')
+
+        if (!path?.startsWith('/')) {
+          errors.push(`Content-Usage path "${path}" must start with a \`/\`.`)
+        }
+        if (!preference.includes('=')) {
+          errors.push(`Content-Usage preference "${preference}" must contain an assignment (e.g., "ai=n").`)
+        }
+      }
+    })
+  }
 }
 
 function matches(pattern: string, path: string): boolean {
@@ -226,11 +265,13 @@ export function asArray(v: any) {
 export function normalizeGroup(group: RobotsGroupInput): RobotsGroupResolved {
   const disallow = asArray(group.disallow) // we can have empty disallow
   const allow = asArray(group.allow).filter(rule => Boolean(rule))
+  const contentUsage = asArray(group.contentUsage).filter(rule => Boolean(rule))
   return <RobotsGroupResolved> {
     ...group,
     userAgent: group.userAgent ? asArray(group.userAgent) : ['*'],
     disallow,
     allow,
+    contentUsage,
     _indexable: !disallow.includes((rule: string) => rule === '/'),
     _rules: [
       ...disallow.filter(Boolean).map(r => ({ pattern: r, allow: false })),
@@ -262,6 +303,10 @@ export function generateRobotsTxt({ groups, sitemaps }: { groups: RobotsGroupRes
     for (const cleanParam of group.cleanParam || [])
       lines.push(`Clean-param: ${cleanParam}`)
 
+    // content signals / AI preferences (see https://datatracker.ietf.org/doc/draft-ietf-aipref-attach/)
+    for (const contentUsage of group.contentUsage || [])
+      lines.push(`Content-Usage: ${contentUsage}`)
+
     lines.push('') // seperator
   }
   // add sitemaps

test/unit/generateRobotsTxt.test.ts

@@ -43,4 +43,34 @@ describe('generateRobotsTxt', () => {
     const generated = generateRobotsTxt(parsed)
     expect(robotsTxt.trim()).toEqual(generated.trim())
   })
+
+  it('content-usage generation', () => {
+    const robotsData = {
+      groups: [
+        {
+          userAgent: ['*'],
+          allow: ['/'],
+          disallow: [],
+          comment: [],
+          contentUsage: [
+            'ai=n',
+            '/public/ train-ai=y',
+            '/restricted/ ai=n train-ai=n',
+          ],
+        },
+      ],
+      sitemaps: ['https://example.com/sitemap.xml'],
+    }
+
+    const generated = generateRobotsTxt(robotsData)
+    expect(generated).toMatchInlineSnapshot(`
+      "User-agent: *
+      Allow: /
+      Content-Usage: ai=n
+      Content-Usage: /public/ train-ai=y
+      Content-Usage: /restricted/ ai=n train-ai=n
+
+      Sitemap: https://example.com/sitemap.xml"
+    `)
+  })
 })

test/unit/robotsTxtParser.test.ts

@@ -13,6 +13,7 @@ describe('robotsTxtParser', () => {
           {
             "allow": [],
             "comment": [],
+            "contentUsage": [],
             "disallow": [
               "/wp-json/",
               "/?s=*",
@@ -27,6 +28,7 @@ describe('robotsTxtParser', () => {
           {
             "allow": [],
             "comment": [],
+            "contentUsage": [],
             "disallow": [
               "/",
             ],
@@ -66,6 +68,7 @@ describe('robotsTxtParser', () => {
               "/api/ui-extensions/",
             ],
             "comment": [],
+            "contentUsage": [],
             "disallow": [
               "/config",
               "/search",
@@ -104,6 +107,7 @@ describe('robotsTxtParser', () => {
               "/api/ui-extensions/",
             ],
             "comment": [],
+            "contentUsage": [],
             "disallow": [
               "/config",
               "/search",
@@ -142,6 +146,7 @@ describe('robotsTxtParser', () => {
               "/api/ui-extensions/",
             ],
             "comment": [],
+            "contentUsage": [],
             "disallow": [
               "/config",
               "/search",
@@ -180,6 +185,7 @@ describe('robotsTxtParser', () => {
               "/api/ui-extensions/",
             ],
             "comment": [],
+            "contentUsage": [],
             "disallow": [
               "/config",
               "/search",
@@ -231,6 +237,7 @@ describe('robotsTxtParser', () => {
           {
             "allow": [],
             "comment": [],
+            "contentUsage": [],
             "disallow": [
               "",
             ],
@@ -257,6 +264,7 @@ describe('robotsTxtParser', () => {
           {
             "allow": [],
             "comment": [],
+            "contentUsage": [],
             "disallow": [
               "/cdn-cgi/challenge-platform/",
             ],
@@ -271,6 +279,7 @@ describe('robotsTxtParser', () => {
               "s /forum/showthread.php",
             ],
             "comment": [],
+            "contentUsage": [],
             "disallow": [
               "",
             ],
@@ -298,6 +307,7 @@ describe('robotsTxtParser', () => {
               "/bar",
             ],
             "comment": [],
+            "contentUsage": [],
             "disallow": [
               "/foo",
             ],
@@ -310,6 +320,7 @@ describe('robotsTxtParser', () => {
               "/boo",
             ],
             "comment": [],
+            "contentUsage": [],
             "disallow": [
               "/baz",
             ],
@@ -320,6 +331,7 @@ describe('robotsTxtParser', () => {
           {
             "allow": [],
             "comment": [],
+            "contentUsage": [],
             "disallow": [
               "/invalid",
             ],
@@ -330,6 +342,7 @@ describe('robotsTxtParser', () => {
           {
             "allow": [],
             "comment": [],
+            "contentUsage": [],
             "disallow": [
               "/star",
             ],
@@ -367,6 +380,7 @@ Unknown: /bar
               "/",
             ],
             "comment": [],
+            "contentUsage": [],
             "disallow": [],
             "userAgent": [
               "*",
@@ -377,4 +391,47 @@ Unknown: /bar
       }
     `)
   })
+
+  it('content-usage directive parsing', () => {
+    const robotsTxt = `
+User-Agent: *
+Allow: /
+Content-Usage: ai=n
+Content-Usage: /public/ train-ai=y
+Content-Usage: /restricted/ ai=n train-ai=n
+    `
+    expect(parseRobotsTxt(robotsTxt)).toMatchInlineSnapshot(`
+      {
+        "errors": [],
+        "groups": [
+          {
+            "allow": [
+              "/",
+            ],
+            "comment": [],
+            "contentUsage": [
+              "ai=n",
+              "/public/ train-ai=y",
+              "/restricted/ ai=n train-ai=n",
+            ],
+            "disallow": [],
+            "userAgent": [
+              "*",
+            ],
+          },
+        ],
+        "sitemaps": [],
+      }
+    `)
+  })
+
+  it('content-usage validation errors', () => {
+    const robotsTxt = `
+User-Agent: *
+Content-Usage: invalid-preference
+Content-Usage: invalid-path ai=n
+Content-Usage: 
+    `
+    expect(parseRobotsTxt(robotsTxt).errors).toEqual([])
+  })
 })