Implements the Content Signals extension of robots.txt

Signed-off-by: Jon Stovell <jonstovell@gmail.com>

Author: Sesquipedalian

Languages/en_US/Help.php

@@ -587,6 +587,10 @@
 		</li>
 	</ul>';
 $helptxt['robots_txt'] = 'The robots.txt file is used to implement the <a href="https://www.rfc-editor.org/rfc/rfc9309.html" class="bbc_link">Robots Exclusion Protocol</a>, a standard used by websites to indicate to search engine spiders and other web robots which portions of the website they are allowed to visit. This file is typically located in your website’s root directory.<br><br>SMF adds some rules to this file in order to guide spiders away from URLs that they should not bother to crawl. This improves efficiency and reduces server load when a spider is crawling your forum.';
+$helptxt['robots_txt_ai_train'] = 'This setting tells artificial intelligence companies whether they are permitted to use the content of your forum to train their AI bots. Unfortunately, there is no way to force AI companies to obey these rules, but well-behaved ones will respect them.';
+$helptxt['robots_txt_search'] = 'This setting tells search engines whether they are permitted to show links to your forum in their search results. Most forums will want this enabled, but if you want your forum not to appear in search results, you can disable it.';
+$helptxt['robots_txt_ai_input'] = 'This setting tells search engines and other web robots whether they are permitted to use the content of your forum to generate AI summaries, etc. Unfortunately, there is no way to force search engines that index your site to obey these rules, but well-behaved ones will respect them.';
+$helptxt['robots_txt_ai_train'] = 'This setting tells artificial intelligence companies whether they are permitted to use the content of your forum to train their AI bots. Unfortunately, there is no way to force AI companies to obey these rules, but well-behaved ones will respect them.';
 
 $helptxt['birthday_email'] = 'Choose the index of the birthday email message to use. A preview will be shown in the Email Subject and Email Body fields.<br><strong>Note:</strong> Selecting this setting does not automatically enable birthday emails. To enable birthday emails use the <a href="{scripturl}?action=admin;area=scheduledtasks;{session_var}={session_id}" target="_blank" rel="noopener">Scheduled Tasks</a> page and enable the birthday email task.';
 $helptxt['pm_bcc'] = 'When sending a personal message you can choose to add a recipient as BCC (Blind Carbon Copy). BCC recipients do not have their identities revealed to the other recipients of the message.';

Languages/en_US/Search.php

@@ -185,6 +185,9 @@
 $txt['robots_txt_info'] = 'Enter the path to your robots.txt file so that SMF can append rules to it.';
 $txt['robots_txt_auto'] = 'Detect path';
 $txt['robots_txt_not_writable'] = 'The robots.txt file is not writable.';
+$txt['robots_txt_search'] = 'Allow search engines to link to the forum in their search results';
+$txt['robots_txt_ai_input'] = 'Allow search engines to generate AI summaries of forum content';
+$txt['robots_txt_ai_train'] = 'Allow forum content to be used for AI training';
 
 $txt['meta_keywords'] = 'Meta keywords associated with forum';
 $txt['meta_keywords_note'] = 'In most situations, this should be left blank.';

Sources/Actions/Admin/SearchEngines.php

@@ -830,16 +830,23 @@ function disableFields()
 			}
 			disableFields();';
 
-		// Now the setting for robots.txt.
+		// Now the settings for robots.txt.
 		$config_vars[] = '';
 
 		if (empty(Config::$modSettings['robots_txt'])) {
+			// Make it easy for the admin to populate the setting.
 			$post_input = '<button class="button floatnone" onclick="document.getElementById(\'robots_txt\').value = ' . Utils::escapeJavaScript(self::detectRobotsTxt()) . '; return false;">' . Lang::getTxt('robots_txt_auto', file: 'Search') . '</button>';
 		} elseif (!is_writable(Config::$modSettings['robots_txt'])) {
+			// Warn if robots.txt is not writable.
 			$invalid = true;
 			$post_input = '<br><span class="error">' . Lang::getTxt('robots_txt_not_writable', file: 'Search') . '</span>';
 		}
 
+		// If robots_txt_search is not set, assume true.
+		if (!isset(Config::$modSettings['robots_txt_search'])) {
+			Config::$modSettings['robots_txt_search'] = true;
+		}
+
 		$config_vars = array_merge($config_vars, [
 			[
 				'text',
@@ -849,6 +856,18 @@ function disableFields()
 				'invalid' => $invalid ?? false,
 				'postinput' => $post_input ?? '',
 			],
+			[
+				'check',
+				'robots_txt_search',
+			],
+			[
+				'check',
+				'robots_txt_ai_input',
+			],
+			[
+				'check',
+				'robots_txt_ai_train',
+			],
 			[
 				'large_text',
 				'meta_keywords',
@@ -1207,38 +1226,99 @@ protected static function addRobotsTxtRules(): void
 		$boardpath = Url::create(Config::$boardurl)->path;
 		$scriptpath = Url::create(Config::$scripturl)->path;
 
+		// Content Signals Policy text.
+		$content_signals_policy = <<<'END'
+			# As a condition of accessing this website, you agree
+			# to abide by the following content signals:
+
+			# (a)  If a content-signal = yes, you may collect
+			# content for the corresponding use.
+			# (b)  If a content-signal = no, you may not collect
+			# content for the corresponding use.
+			# (c)  If the website operator does not include a
+			# content signal for a corresponding use, the website
+			# operator neither grants nor restricts permission via
+			# content signal with respect to the corresponding use.
+
+			# The content signals and their meanings are:
+
+			# search: building a search index and providing search
+			# results (e.g., returning hyperlinks and short
+			# excerpts from your website's contents).  Search does
+			# not include providing AI-generated search summaries.
+			# ai-input: inputting content into one or more AI
+			# models (e.g., retrieval augmented generation,
+			# grounding, or other real-time taking of content for
+			# generative AI search answers).
+			# ai-train: training or fine-tuning AI models.
+
+			# ANY RESTRICTIONS EXPRESSED VIA CONTENT SIGNALS ARE
+			# EXPRESS RESERVATIONS OF RIGHTS UNDER ARTICLE 4 OF THE
+			# EUROPEAN UNION DIRECTIVE 2019/790 ON COPYRIGHT AND
+			# RELATED RIGHTS IN THE DIGITAL SINGLE MARKET.
+
+
+			END;
+
 		// Define the rules we want to include.
+		//
+		// Top level keys are user agent strings.
+		// For example, '*' means 'User-Agent: *', i.e. all robots.
+		// If a mod wants to add specific rules for certain robots, new keys
+		// can be added to the $rules array targeting those specific robots.
+		//
+		// Within each user agent's ruleset, keys are path patterns as described
+		// in RFC 9309. There are three possible values:
+		//
+		//  - false
+		//        The user agent is not allowed to crawl the indicated path.
+		//
+		//  - true
+		//        The user agent is allowed to crawl the indicated path with no
+		//        restrictions on how the content may be used.
+		//
+		//  - an array containing 'search', 'ai-train', and 'ai-input' elements,
+		//    each of which takes a boolean value.
+		//        The user agent is allowed to crawl the indicated path, but a
+		//        Content-Signal line will be inserted to indicate restrictions
+		//        on how the content may be used.
 		$rules = [
 			'*' => [
-				'allow' => [],
-				'disallow' => [
-					// Frequenty occurring non-canonical URLs (both normal and queryless)
-					$boardpath . '/*PHPSESSID=',
-					$boardpath . '/*;topicseen',
-					$boardpath . '/*.msg',
-					$boardpath . '/*.new',
-					$boardpath . '/*.from',
-					$boardpath . '/msgs/',
-					// Normal URLs of actions that always set Utils::$context['robot_no_index'] to true
-					$scriptpath . '?action=admin',
-					$scriptpath . '?action=credits',
-					$scriptpath . '?action=moderate',
-					$scriptpath . '?action=post',
-					$scriptpath . '?action=printpage',
-					$scriptpath . '?action=reminder',
-					$scriptpath . '?action=reporttm',
-					$scriptpath . '?action=search',
-					$scriptpath . '?action=who',
-					// Queryless URLs of actions that always set Utils::$context['robot_no_index'] to true
-					$boardpath . '/*/credits',
-					$boardpath . '/*/moderate',
-					$boardpath . '/*/post',
-					$boardpath . '/*/printpage',
-					$boardpath . '/*/reminder',
-					$boardpath . '/*/reporttm',
-					$boardpath . '/*/search',
-					$boardpath . '/*/who',
+				// Allow robots to crawl the forum, subject to the Content-Signal policy.
+				$boardpath . '/' => [
+					// If robots_txt_search is not set, assume true.
+					'search' => !empty(Config::$modSettings['robots_txt_search'] ?? true),
+					// If the AI settings are not set, assume false.
+					'ai-train' => !empty(Config::$modSettings['robots_txt_ai_train']),
+					'ai-input' => !empty(Config::$modSettings['robots_txt_ai_input']),
 				],
+
+				// Disallow frequently occurring non-canonical URLs.
+				$boardpath . '/*PHPSESSID=' => false,
+				$boardpath . '/*;topicseen' => false,
+				$boardpath . '/*.msg' => false,
+				$boardpath . '/*.new' => false,
+				$boardpath . '/*.from' => false,
+				$boardpath . '/msgs/' => false,
+
+				// Disallow actions that always set Utils::$context['robot_no_index'] to true.
+				$scriptpath . '?action=admin' => false,
+				$scriptpath . '?action=credits' => false,
+				$scriptpath . '?action=moderate' => false,
+				$scriptpath . '?action=post' => false,
+				$scriptpath . '?action=printpage' => false,
+				$scriptpath . '?action=reminder' => false,
+				$scriptpath . '?action=reporttm' => false,
+				$scriptpath . '?action=search' => false,
+				$scriptpath . '?action=who' => false,
+				$boardpath . '/*/credits' => false,
+				$boardpath . '/*/moderate' => false,
+				$boardpath . '/*/post' => false,
+				$boardpath . '/*/printpage' => false,
+				$boardpath . '/*/reminder' => false,
+				$boardpath . '/*/reporttm' => false,
+				$boardpath . '/*/search' => false,
+				$boardpath . '/*/who' => false,
 			],
 		];
 
@@ -1250,61 +1330,49 @@ protected static function addRobotsTxtRules(): void
 		if (is_file(Config::$modSettings['robots_txt'])) {
 			$hash = md5_file(Config::$modSettings['robots_txt']);
 
-			$user_agents_in_group = [];
-			$current_user_agent = '';
-			$insert = false;
+			$user_agents = [];
+			$in_user_agents = false;
 
 			// Keep all existing content and filter out anything in $rules that already exists.
 			foreach (file(Config::$modSettings['robots_txt']) as $line) {
 				// Found a new user agent line.
 				if (preg_match('/^\h*user-agent:\h*([^\n]+)/i', $line, $matches)) {
-					$user_agents_in_group[] = $matches[1];
-					$current_user_agent = $matches[1];
-
-					if ($insert === null) {
-						$insert = true;
+					if (!$in_user_agents) {
+						$user_agents = [];
 					}
-				} elseif (preg_match('/^\h*($|#)/i', $line)) {
-					$insert = true;
-				} else {
-					$insert = null;
-				}
 
-				// Insert our rules before comments, blank lines, or the start
-				// of a new user agent group, but only if user agent that these
-				// rules are for was the only one in its group.
-				if (!empty($insert) && \count($user_agents_in_group) === 1) {
-					foreach ($user_agents_in_group as $user_agent) {
-						if (!isset($rules[$user_agent])) {
-							continue;
-						}
-
-						foreach ($rules[$user_agent] as $type => $patterns) {
-							foreach ($patterns as $pattern) {
-								$new_content[] = ucfirst($type) . ': ' . $pattern . "\n";
-							}
-						}
-
-						// Don't do the same rules twice.
-						unset($rules[$user_agent]);
-					}
-
-					$insert = false;
+					$user_agents[] = $matches[1];
+					$in_user_agents = true;
 				}
 
 				// Append this line.
 				$new_content[] = $line;
 
 				// Filter out anything in $rules that already exists.
-				if (preg_match('/^\h*((?:dis)?allow)\h*:\h*([^\n]+)/i', $line, $matches)) {
-					$type = strtolower($matches[1]);
-					$pattern = $matches[2];
-
-					if (isset($rules[$current_user_agent][$type])) {
-						$rules[$current_user_agent][$type] = array_diff(
-							$rules[$current_user_agent][$type],
-							[$pattern],
-						);
+				if (preg_match('/^\h*((?:dis)?allow|content-signal)\h*:\h*([^\n]+)/i', $line, $matches)) {
+					$in_user_agents = false;
+					$allowed = strtolower($matches[1]) !== 'disallow';
+
+					if (strtolower($matches[1]) !== 'content-signal') {
+						$pattern = $matches[2];
+						$signal = null;
+					} else {
+						[$pattern, $signal] = preg_split('/\h+/', $matches[2], 2);
+						parse_str(preg_replace('/\h*,\h*/', '&', $signal), $signal);
+						$signal = array_map(fn($arg) => $arg === 'yes', $signal);
+					}
+
+					foreach ($user_agents as $user_agent) {
+						if (!isset($rules[$user_agent][$pattern])) {
+							continue;
+						}
+
+						if (
+							$rules[$user_agent][$pattern] === $allowed
+							|| $rules[$user_agent][$pattern] === $signal
+						) {
+							unset($rules[$user_agent][$pattern]);
+						}
 					}
 				}
 			}
@@ -1342,19 +1410,44 @@ protected static function addRobotsTxtRules(): void
 
 		// Append any new rules that haven't already been inserted.
 		foreach ($rules as $user_agent => $rule_parts) {
+			if (empty($rule_parts)) {
+				continue;
+			}
+
 			$new_content[] = "\n";
 			$new_content[] = 'User-agent: ' . $user_agent . "\n";
 
-			foreach ($rule_parts as $type => $patterns) {
-				foreach ($patterns as $pattern) {
-					$new_content[] = ucfirst($type) . ': ' . $pattern . "\n";
+			foreach ($rule_parts as $pattern => $rule) {
+				if (\is_array($rule)) {
+					switch (array_filter($rule)) {
+						// If all uses are allowed, then allow the whole thing.
+						case $rule:
+							$rule = true;
+							break;
+
+						// If all uses are disallowed, then disallow the whole thing.
+						case []:
+							$rule = false;
+							break;
+
+						// Indicate what types of use are allowed or disallowed.
+						default:
+							$new_content[] = 'Content-Signal: ' . $pattern . ' ' . implode(', ', array_map(fn($k, $v) => $k . '=' . ($v ? 'yes' : 'no'), array_keys($rule), $rule)) . "\n";
+							break;
+					}
 				}
+
+				$new_content[] = (empty($rule) ? 'Disallow: ' : 'Allow: ') . $pattern . "\n";
 			}
 		}
 
 		// Finalize the content.
 		$new_content = trim(implode('', $new_content)) . "\n";
 
+		if (!str_contains($new_content, $content_signals_policy)) {
+			$new_content = $content_signals_policy . $new_content;
+		}
+
 		// If nothing changed, bail out.
 		if (isset($hash) && md5($new_content) === $hash) {
 			return;

Sources/Db/Schema/v3_0/Settings.php

@@ -630,6 +630,10 @@ class Settings extends Table
 			'variable' => 'reserveWord',
 			'value' => '0',
 		],
+		[
+			'variable' => 'robots_txt_search',
+			'value' => '1',
+		],
 		[
 			'variable' => 'samesiteCookies',
 			'value' => 'lax',