Skip to content

Commit 61d6d34

Browse files
harlan-zwharlan-zw
authored
feat: type-safe object syntax for useRobotsRule (#214)
1 parent 1510d3f commit 61d6d34

File tree

15 files changed

+809
-23
lines changed

15 files changed

+809
-23
lines changed

‎.playground/pages/ai-test.vue‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
<script setup>
2+
const robotRule = useRobotsRule()
3+
4+
function blockAI() {
5+
robotRule.value = { noai: true }
6+
}
7+
8+
function blockImageAI() {
9+
robotRule.value = { noimageai: true }
10+
}
11+
12+
function blockBoth() {
13+
robotRule.value = { noai: true, noimageai: true }
14+
}
15+
16+
function setLargePreview() {
17+
robotRule.value = { 'max-image-preview': 'large' }
18+
}
19+
20+
function setStandardPreview() {
21+
robotRule.value = { 'max-image-preview': 'standard' }
22+
}
23+
24+
function setNoPreview() {
25+
robotRule.value = { 'max-image-preview': 'none' }
26+
}
27+
28+
function setLimitedSnippet() {
29+
robotRule.value = { 'max-snippet': 120 }
30+
}
31+
32+
function setUnlimitedSnippet() {
33+
robotRule.value = { 'max-snippet': -1 }
34+
}
35+
36+
function setCustomCombination() {
37+
// Combine multiple directives
38+
robotRule.value = {
39+
'index': true,
40+
'follow': true,
41+
'noai': true,
42+
'max-image-preview': 'standard',
43+
'max-snippet': 150,
44+
'max-video-preview': 30,
45+
}
46+
}
47+
48+
function allowAll() {
49+
robotRule.value = true
50+
}
51+
52+
// Set initial rule with preview settings
53+
robotRule.value = {
54+
'max-image-preview': 'large',
55+
'max-snippet': -1,
56+
}
57+
</script>
58+
59+
<template>
60+
<div>
61+
<h1>Robot Directives Test Page</h1>
62+
<p>This page demonstrates various robot directives including non-standard ones.</p>
63+
64+
<div>
65+
<h2>Current robot rule: {{ robotRule }}</h2>
66+
67+
<h3>AI Directives (Non-standard)</h3>
68+
<button @click="blockAI">
69+
Block AI Crawlers
70+
</button>
71+
<button @click="blockImageAI">
72+
Block AI Image Crawlers
73+
</button>
74+
<button @click="blockBoth">
75+
Block AI (Both)
76+
</button>
77+
78+
<h3>Preview Directives</h3>
79+
<button @click="setLargePreview">
80+
Large Image Preview
81+
</button>
82+
<button @click="setStandardPreview">
83+
Standard Image Preview
84+
</button>
85+
<button @click="setNoPreview">
86+
No Image Preview
87+
</button>
88+
<button @click="setLimitedSnippet">
89+
Limited Text Snippet (120 chars)
90+
</button>
91+
<button @click="setUnlimitedSnippet">
92+
Unlimited Snippet
93+
</button>
94+
95+
<h3>Combined Examples</h3>
96+
<button @click="setCustomCombination">
97+
Custom Combination
98+
</button>
99+
<button @click="allowAll">
100+
Allow All (Default)
101+
</button>
102+
</div>
103+
</div>
104+
</template>

‎docs/content/2.guides/1.disable-page-indexing.md‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,19 @@ The [useRobotsRule](/docs/robots/api/use-robots-rule) composable provides a reac
3636
```ts
3737
import { useRobotsRule } from '#imports'
3838

39+
// Using string syntax
3940
const rule = useRobotsRule()
4041
rule.value = 'noindex, nofollow'
42+
43+
// Using object syntax (recommended)
44+
useRobotsRule({ noindex: true, nofollow: true })
45+
46+
// Combining with AI-specific directives
47+
useRobotsRule({
48+
noindex: true,
49+
noai: true, // Prevent AI crawlers from using content
50+
noimageai: true // Prevent AI crawlers from using images
51+
})
4152
```
4253

4354
## Route Rules

‎docs/content/2.guides/2.route-rules.md‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ You can provide the following rules:
99

1010
- `{ robots: false }`{lang="json"} - Will disable the route from being indexed using the [robotsDisabledValue](/docs/robots/api/config#robotsdisabledvalue) config.
1111
- `{ robots: '<rule>' }`{lang="json"} - Will add the provided string as the robots rule
12+
- `{ robots: { /* directives */ } }`{lang="json"} - Will use object syntax to define robot directives
1213

1314
The rules are applied using the following logic:
1415
- `X-Robots-Tag` header - SSR only,
@@ -39,6 +40,23 @@ export default defineNuxtConfig({
3940
'/secret/visible': { robots: true },
4041
// use the `robots` rule if you need finer control
4142
'/custom-robots': { robots: 'index, follow' },
43+
// use object syntax for more complex rules
44+
'/ai-protected': {
45+
robots: {
46+
index: true,
47+
noai: true,
48+
noimageai: true
49+
}
50+
},
51+
// control search result previews
52+
'/limited-preview': {
53+
robots: {
54+
index: true,
55+
'max-image-preview': 'standard',
56+
'max-snippet': 100,
57+
'max-video-preview': 15
58+
}
59+
}
4260
}
4361
})
4462
```

‎docs/content/3.api/0.use-robots-rule.md‎

Lines changed: 69 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,9 @@ description: A reactive way to access and set the robots rule.
55

66
## Introduction
77

8-
**Type:** `function useRobotsRule(rule?: MaybeRef<boolean | string>): Ref<string>`{lang="ts"}
8+
**Type:** `function useRobotsRule(rule?: MaybeRef<boolean | string | Partial<RobotDirectives>>): Ref<string>`{lang="ts"}
99

10-
View and control the robots rule using a simple reactivity API.
10+
View and control the robots rule using a simple reactivity API. Supports standard directives (index, noindex, follow, nofollow) and non-standard directives like noai and noimageai.
1111

1212
It's recommended to use this composable when you need to dynamically change the robots rule at runtime. For example when a user changes their profile from private to public.
1313

@@ -35,6 +35,27 @@ import { useRobotsRule } from '#imports'
3535
const rule = useRobotsRule(true) // does not do anything, just returns the value
3636
```
3737

38+
## Available Directives
39+
40+
When using the object syntax, you can use the following directives:
41+
42+
### Standard Directives
43+
- `index`: Allow search engines to index the page
44+
- `noindex`: Prevent search engines from indexing the page
45+
- `follow`: Allow search engines to follow links on the page
46+
- `nofollow`: Prevent search engines from following links on the page
47+
- `none`: Equivalent to `noindex, nofollow`
48+
- `all`: Equivalent to `index, follow`
49+
50+
### Non-Standard Directives
51+
- `noai`: Request AI crawlers not to use content for training
52+
- `noimageai`: Request AI crawlers not to use images for training
53+
54+
### Preview Control Directives
55+
- `max-image-preview`: Controls image preview size (`'none'`, `'standard'`, or `'large'`)
56+
- `max-snippet`: Controls text snippet length in characters (use `-1` for no limit)
57+
- `max-video-preview`: Controls video preview length in seconds (use `-1` for no limit)
58+
3859
## Usage
3960

4061
**Accessing the rule:**
@@ -66,3 +87,49 @@ const rule = useRobotsRule()
6687
rule.value = 'index, nofollow'
6788
// Ref<'index, nofollow'>
6889
```
90+
91+
**Setting the rule - object syntax:**
92+
93+
```ts
94+
import { useRobotsRule } from '#imports'
95+
96+
// Using object syntax for directives
97+
useRobotsRule({ noindex: true, nofollow: true })
98+
// Ref<'noindex, nofollow'>
99+
100+
// Combining standard and non-standard directives
101+
useRobotsRule({ index: true, noai: true, noimageai: true })
102+
// Ref<'index, noai, noimageai'>
103+
104+
// Only true values are included
105+
useRobotsRule({ index: true, follow: false, noai: true })
106+
// Ref<'index, noai'>
107+
108+
// Empty object defaults to enabled value
109+
useRobotsRule({})
110+
// Ref<'index, follow, max-image-preview:large, max-snippet:-1, max-video-preview:-1'>
111+
```
112+
113+
**Setting the rule - with max-* directives:**
114+
115+
```ts
116+
import { useRobotsRule } from '#imports'
117+
118+
// Control search result preview settings
119+
useRobotsRule({
120+
index: true,
121+
'max-image-preview': 'large', // 'none', 'standard', or 'large'
122+
'max-snippet': 150, // number of characters
123+
'max-video-preview': 30 // seconds of video preview
124+
})
125+
// Ref<'index, max-image-preview:large, max-snippet:150, max-video-preview:30'>
126+
127+
// Disable all previews
128+
useRobotsRule({
129+
index: true,
130+
'max-image-preview': 'none',
131+
'max-snippet': 0,
132+
'max-video-preview': 0
133+
})
134+
// Ref<'index, max-image-preview:none, max-snippet:0, max-video-preview:0'>
135+
```

‎src/module.ts‎

Lines changed: 31 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import { setupDevToolsUI } from './devtools'
2222
import { resolveI18nConfig, splitPathForI18nLocales } from './i18n'
2323
import { isNuxtGenerate, resolveNitroPreset } from './kit'
2424
import { logger } from './logger'
25+
import { formatMaxImagePreview, formatMaxSnippet, formatMaxVideoPreview, ROBOT_DIRECTIVE_VALUES } from './runtime/const'
2526
import {
2627
normaliseRobotsRouteRule,
2728
} from './runtime/server/nitro'
@@ -200,8 +201,8 @@ export default defineNuxtModule<ModuleOptions>({
200201
header: true,
201202
metaTag: true,
202203
cacheControl: 'max-age=14400, must-revalidate',
203-
robotsEnabledValue: 'index, follow, max-image-preview:large, max-snippet:-1, max-video-preview:-1',
204-
robotsDisabledValue: 'noindex, nofollow',
204+
robotsEnabledValue: ROBOT_DIRECTIVE_VALUES.enabled,
205+
robotsDisabledValue: ROBOT_DIRECTIVE_VALUES.disabled,
205206
disallowNonIndexableRoutes: false,
206207
robotsTxt: true,
207208
botDetection: true,
@@ -351,9 +352,34 @@ export default defineNuxtModule<ModuleOptions>({
351352
nuxt.hooks.hook('content:file:afterParse', (ctx) => {
352353
if (typeof ctx.content.robots !== 'undefined') {
353354
let rule = ctx.content.robots
354-
if (typeof rule === 'boolean' || !rule) {
355+
if (typeof rule === 'boolean') {
355356
rule = rule ? config.robotsEnabledValue : config.robotsDisabledValue
356357
}
358+
else if (typeof rule === 'object' && rule !== null) {
359+
const directives: string[] = []
360+
for (const [key, value] of Object.entries(rule)) {
361+
if (value === false || value === null || value === undefined)
362+
continue
363+
364+
// Handle boolean directives
365+
if (key in ROBOT_DIRECTIVE_VALUES && typeof value === 'boolean' && value) {
366+
directives.push(ROBOT_DIRECTIVE_VALUES[key as keyof typeof ROBOT_DIRECTIVE_VALUES])
367+
}
368+
// Handle max-image-preview
369+
else if (key === 'max-image-preview' && typeof value === 'string') {
370+
directives.push(formatMaxImagePreview(value as 'none' | 'standard' | 'large'))
371+
}
372+
// Handle max-snippet
373+
else if (key === 'max-snippet' && typeof value === 'number') {
374+
directives.push(formatMaxSnippet(value))
375+
}
376+
// Handle max-video-preview
377+
else if (key === 'max-video-preview' && typeof value === 'number') {
378+
directives.push(formatMaxVideoPreview(value))
379+
}
380+
}
381+
rule = directives.join(', ') || config.robotsEnabledValue
382+
}
357383
// add route rule for the path
358384
ctx.content.seo = ctx.content.seo || {}
359385
// @ts-expect-error runtime type
@@ -485,13 +511,13 @@ export default defineNuxtModule<ModuleOptions>({
485511
_robotsPatternMap?: Map<string, import('${typesPath}').PatternMapValue>
486512
}
487513
interface NitroRouteRules {
488-
robots?: boolean | string | {
514+
robots?: import('${typesPath}').RobotsValue | {
489515
indexable: boolean
490516
rule: string
491517
}
492518
}
493519
interface NitroRouteConfig {
494-
robots?: boolean | string | {
520+
robots?: import('${typesPath}').RobotsValue | {
495521
indexable: boolean
496522
rule: string
497523
}

0 commit comments

Comments
 (0)