Merge pull request #8984 from Sesquipedalian/3.0/robots_txt_content_signals

[3.0] Implements the Content Signals extension of robots.txt

Author: Sesquipedalian

Languages/en_US/Admin.php

@@ -234,7 +234,7 @@
 $txt['admin_log_desc'] = 'Lists administrative tasks that have been performed by admins of your forum.';
 $txt['moderation_log'] = 'Moderation Log';
 $txt['moderation_log_desc'] = 'Lists moderation activities that have been performed by moderators on your forum.';
-$txt['spider_log_desc'] = 'Review the entries related to search engine spider activity on your forum.';
+$txt['spider_log_desc'] = 'Review the entries related to search engine and robot activity on your forum.';
 $txt['log_settings_desc'] = 'Use these options to configure how logging works on your forum.';
 $txt['modlog_enabled'] = 'Enable the moderation log';
 $txt['adminlog_enabled'] = 'Enable the administration log';
@@ -747,7 +747,7 @@
 $txt['admin_browse_register_new'] = 'Register new member';
 
 $txt['search_engines'] = 'Search Engines';
-$txt['spider_logs'] = 'Spider Log';
+$txt['spider_logs'] = 'Robot Log';
 $txt['spider_stats'] = 'Stats';
 
 $txt['paid_subscriptions'] = 'Paid Subscriptions';

Languages/en_US/General.php

@@ -1103,15 +1103,15 @@
 $txt['ignoring_user'] = 'You are ignoring this user.';
 $txt['show_ignore_user_post'] = 'Show me the post.';
 
-$txt['spider'] = 'Spider';
-$txt['spiders'] = 'Spiders';
+$txt['spider'] = 'Robot';
+$txt['spiders'] = 'Robots';
 $txt['spider_plural'] = '{0, plural,
-	one {spider}
-	other {spiders}
+	one {robot}
+	other {robots}
 }';
 $txt['number_of_spiders'] = '{0, plural,
-	one {# spider}
-	other {# spiders}
+	one {# robot}
+	other {# robots}
 }';
 
 $txt['downloads'] = 'Downloads';

Languages/en_US/Help.php

@@ -565,28 +565,31 @@
 	</ul>';
 
 $helptxt['spider_mode'] = 'Sets the logging level.<br>
-Standard - Logs minimal spider activity.<br>
+Standard - Logs minimal robot activity.<br>
 Moderate - Provides more accurate statistics.<br>
 Aggressive - Same as &quot;Moderate&quot;, but logs data about each page visited.';
 
-$helptxt['spider_group'] = 'By selecting a restrictive group, when a guest is detected as a search crawler it will automatically be assigned any &quot;deny&quot; permissions of this group in addition to the normal permissions of a guest. You can use this to provide lesser access to a search engine than you would a normal guest. You might for example wish to create a new group called &quot;Spiders&quot; and select that here. You could then deny permission for that group to view profiles to stop spiders indexing your members profiles.<br>Note: Spider detection is not perfect and can be simulated by users so this feature is not guaranteed to restrict content only to those search engines you have added.';
-$helptxt['show_spider_online'] = 'This setting allows you to select whether spiders should be listed in the &quot;Who’s Online&quot; list on the board index and &quot;Who’s Online&quot; page. The options are:
+$helptxt['spider_group'] = 'By selecting a restrictive group, when a guest is detected as a search crawler it will automatically be assigned any &quot;deny&quot; permissions of this group in addition to the normal permissions of a guest. You can use this to provide lesser access to a search engine than you would a normal guest. You might for example wish to create a new group called &quot;Robots&quot; and select that here. You could then deny permission for that group to view profiles to stop robots indexing your members profiles.<br>Note: Robot detection is not perfect and can be simulated by users so this feature is not guaranteed to restrict content only to those search engines you have added.';
+$helptxt['show_spider_online'] = 'This setting allows you to select whether robots should be listed in the &quot;Who’s Online&quot; list on the board index and &quot;Who’s Online&quot; page. The options are:
 	<ul class="normallist">
 		<li>
 			<strong>Not at All</strong><br>
-			Spiders will simply appear as guests to all users.
+			Robots will simply appear as guests to all users.
 		</li><li>
-			<strong>Show Spider Quantity</strong><br>
-			The Board Index will display the number of spiders currently visiting the forum.
+			<strong>Show Robot Quantity</strong><br>
+			The Board Index will display the number of robots currently visiting the forum.
 		</li><li>
-			<strong>Show Spider Names</strong><br>
-			Each spider name will be revealed, so users can see how many of each spider is currently visiting the forum - this takes effect in both the Board Index and Who’s Online page.
+			<strong>Show Robot Names</strong><br>
+			Each robot name will be revealed, so users can see how many of each robot is currently visiting the forum - this takes effect in both the Board Index and Who’s Online page.
 		</li><li>
-			<strong>Show Spider Names - Admin Only</strong><br>
-			As above except only Administrators can see spider status - to all other users spiders appear as guests.
+			<strong>Show Robot Names - Admin Only</strong><br>
+			As above except only Administrators can see robot status - to all other users robots appear as guests.
 		</li>
 	</ul>';
-$helptxt['robots_txt'] = 'The robots.txt file is used to implement the <a href="https://www.rfc-editor.org/rfc/rfc9309.html" class="bbc_link">Robots Exclusion Protocol</a>, a standard used by websites to indicate to search engine spiders and other web robots which portions of the website they are allowed to visit. This file is typically located in your website’s root directory.<br><br>SMF adds some rules to this file in order to guide spiders away from URLs that they should not bother to crawl. This improves efficiency and reduces server load when a spider is crawling your forum.';
+$helptxt['robots_txt'] = 'The robots.txt file is used to implement the <a href="https://www.rfc-editor.org/rfc/rfc9309.html" class="bbc_link">Robots Exclusion Protocol</a>, a standard used by websites to indicate to search engine spiders and other web robots which portions of the website they are allowed to visit. This file is typically located in your website’s root directory.<br><br>SMF adds some rules to this file in order to guide robots away from URLs that they should not bother to visit. This improves efficiency and reduces server load when a robot is indexing your forum.';
+$helptxt['robots_txt_search'] = 'This setting tells search engines whether they are permitted to show links to your forum in their search results. Most forums will want this enabled, but if you want your forum not to appear in search results, you can disable it.';
+$helptxt['robots_txt_ai_input'] = 'This setting tells search engines and other web robots whether they are permitted to use the content of your forum to generate AI summaries, etc. Unfortunately, there is no way to force search engines that index your site to obey these rules, but well-behaved ones will respect them.';
+$helptxt['robots_txt_ai_train'] = 'This setting tells artificial intelligence companies whether they are permitted to use the content of your forum to train their AI bots. Unfortunately, there is no way to force AI companies to obey these rules, but well-behaved ones will respect them.';
 
 $helptxt['birthday_email'] = 'Choose the index of the birthday email message to use. A preview will be shown in the Email Subject and Email Body fields.<br><strong>Note:</strong> Selecting this setting does not automatically enable birthday emails. To enable birthday emails use the <a href="{scripturl}?action=admin;area=scheduledtasks;{session_var}={session_id}" target="_blank" rel="noopener">Scheduled Tasks</a> page and enable the birthday email task.';
 $helptxt['pm_bcc'] = 'When sending a personal message you can choose to add a recipient as BCC (Blind Carbon Copy). BCC recipients do not have their identities revealed to the other recipients of the message.';

Languages/en_US/Search.php

@@ -124,44 +124,44 @@
 $txt['search_example'] = '<em>e.g.</em> Orwell "Animal Farm" -movie';
 
 $txt['search_engines_description'] = 'From this area you can decide in what detail you wish to track search engines as they index your forum as well as review search engine logs.';
-$txt['spider_mode'] = 'Search Engine Tracking level';
-$txt['spider_mode_note'] = 'Note that logging of every spider action only occurs if tracking is set to either "moderate" or "aggressive". Detail of every spider’s action is only logged if tracking is set to "aggressive".';
+$txt['spider_mode'] = 'Search Engine and Robot Tracking level';
+$txt['spider_mode_note'] = 'Note that logging of every robot action only occurs if tracking is set to either "moderate" or "aggressive". Details about every robot action are only logged if tracking is set to "aggressive".';
 $txt['spider_mode_off'] = 'Disabled';
 $txt['spider_mode_standard'] = 'Standard';
 $txt['spider_mode_high'] = 'Moderate';
 $txt['spider_mode_vhigh'] = 'Aggressive';
-$txt['spider_settings_desc'] = 'You can change settings for spider tracking from this page. Note, if you wish to enable automatic pruning of the hit logs you can set this up <a href="%1$s">here</a>';
+$txt['spider_settings_desc'] = 'You can change settings for robot tracking from this page. Note, if you wish to enable automatic pruning of the hit logs you can set this up <a href="%1$s">here</a>';
 
 $txt['spider_group'] = 'Apply restrictive permissions from group';
-$txt['spider_group_note'] = 'To enable you to stop spiders indexing some pages.';
+$txt['spider_group_note'] = 'To enable you to stop robots indexing some pages.';
 $txt['spider_group_none'] = 'Disabled';
 
-$txt['show_spider_online'] = 'Show spiders in the online list';
+$txt['show_spider_online'] = 'Show robots in the online list';
 $txt['show_spider_online_no'] = 'Not at all';
-$txt['show_spider_online_summary'] = 'Show spider quantity';
-$txt['show_spider_online_detail'] = 'Show spider names';
-$txt['show_spider_online_detail_admin'] = 'Show spider names - admin only';
+$txt['show_spider_online_summary'] = 'Show robot quantity';
+$txt['show_spider_online_detail'] = 'Show robot names';
+$txt['show_spider_online_detail_admin'] = 'Show robot names - admin only';
 
-$txt['spider_name'] = 'Spider Name';
+$txt['spider_name'] = 'Robot Name';
 $txt['spider_last_seen'] = 'Last Seen';
 $txt['spider_last_never'] = 'Never';
 $txt['spider_agent'] = 'User Agent';
 $txt['spider_ip_info'] = 'IP Addresses';
-$txt['spiders_add'] = 'Add new Spider';
-$txt['spiders_edit'] = 'Edit Spider';
+$txt['spiders_add'] = 'Add new robot';
+$txt['spiders_edit'] = 'Edit robot';
 $txt['spiders_remove_selected'] = 'Remove selected';
-$txt['spider_remove_selected_confirm'] = 'Are you sure you want to remove these spiders?-n-All associated statistics will also be deleted!';
-$txt['spiders_no_entries'] = 'There are currently no spiders configured.';
+$txt['spider_remove_selected_confirm'] = 'Are you sure you want to remove these robots?-n-All associated statistics will also be deleted!';
+$txt['spiders_no_entries'] = 'There are currently no robots configured.';
 
-$txt['add_spider_desc'] = 'From this page you can edit the parameters against which a spider is categorized. If a guest’s user agent/IP address matches those entered below it will be detected as a search engine spider and tracked as per the forum preferences.';
-$txt['spider_name_desc'] = 'Name by which the spider will be referred.';
-$txt['spider_agent_desc'] = 'User agent associated with this spider.';
-$txt['spider_ip_info_desc'] = 'Comma separated list of IP addresses associated with this spider.';
+$txt['add_spider_desc'] = 'From this page you can edit the parameters against which a robot is categorized. If a guest’s user agent or IP address match those entered below, the guest will be detected as a robot and tracked as per the forum preferences.';
+$txt['spider_name_desc'] = 'Name by which the robot will be referred.';
+$txt['spider_agent_desc'] = 'User agent associated with this robot.';
+$txt['spider_ip_info_desc'] = 'Comma separated list of IP addresses associated with this robot.';
 
 $txt['spider_time'] = 'Time';
 $txt['spider_viewing'] = 'Viewing';
-$txt['spider_logs_empty'] = 'There are currently no spider log entries.';
-$txt['spider_logs_info'] = 'Note that logging of every spider action only occurs if tracking is set to either "moderate" or "aggressive". Detail of every spider’s action is only logged if tracking is set to "aggressive".';
+$txt['spider_logs_empty'] = 'There are currently no robot log entries.';
+$txt['spider_logs_info'] = 'Note that logging of every robot action only occurs if tracking is set to either "moderate" or "aggressive". Details about every robot action are only logged if tracking is set to "aggressive".';
 $txt['spider_disabled'] = 'Disabled';
 $txt['spider_log_empty_log'] = 'Clear Log';
 $txt['spider_log_empty_log_confirm'] = 'Are you sure you want to completely clear the log';
@@ -172,19 +172,22 @@
 $txt['spider_logs_delete_submit'] = 'Delete';
 
 // argument: an HTML <input> element.
-$txt['spider_stats_delete_older'] = 'Delete all spider statistics from spiders not seen in {input} days.';
+$txt['spider_stats_delete_older'] = 'Delete all statistics from robots not seen in {input} days.';
 
 // Don't use entities in the below string.
 $txt['spider_logs_delete_confirm'] = 'Are you sure you wish to empty out all log entries?';
 
 $txt['spider_stats_select_month'] = 'Jump to Month';
 $txt['spider_stats_page_hits'] = 'Page Hits';
-$txt['spider_stats_no_entries'] = 'There are currently no spider statistics available.';
+$txt['spider_stats_no_entries'] = 'There are currently no robot statistics available.';
 
 $txt['robots_txt'] = 'Add SMF rules to robots.txt';
 $txt['robots_txt_info'] = 'Enter the path to your robots.txt file so that SMF can append rules to it.';
 $txt['robots_txt_auto'] = 'Detect path';
 $txt['robots_txt_not_writable'] = 'The robots.txt file is not writable.';
+$txt['robots_txt_search'] = 'Allow search engines to link to the forum in their search results';
+$txt['robots_txt_ai_input'] = 'Allow search engines to generate AI summaries of forum content';
+$txt['robots_txt_ai_train'] = 'Allow forum content to be used for AI training';
 
 $txt['meta_keywords'] = 'Meta keywords associated with forum';
 $txt['meta_keywords_note'] = 'In most situations, this should be left blank.';

Languages/en_US/Who.php

@@ -13,9 +13,9 @@
 $txt['who_show'] = 'Show ';
 $txt['who_show_members_only'] = 'Members only';
 $txt['who_show_guests_only'] = 'Guests only';
-$txt['who_show_spiders_only'] = 'Spiders only';
+$txt['who_show_spiders_only'] = 'Robots only';
 $txt['who_show_all'] = 'Everyone';
-$txt['who_no_online_spiders'] = 'There are currently no spiders online.';
+$txt['who_no_online_spiders'] = 'There are currently no robots online.';
 $txt['who_no_online_guests'] = 'There are currently no guests online.';
 $txt['who_no_online_members'] = 'There are currently no members online.';
 $txt['who_guest_login'] = 'User has been taken to the login page.';