Merge pull request #990 from 10up/feature/736

Add feature to block AI data scraping bots

Author: dkotter

.github/workflows/vipcs.yml

@@ -33,6 +33,6 @@ jobs:
         with:
           standard: WordPress-VIP-Go
           use_local_config: false
-          enable_warnings: true
+          enable_warnings: false
           only_changed_lines: false
           excludes: tests

README.md

@@ -34,6 +34,7 @@ Tap into leading cloud-based services like [OpenAI](https://openai.com/), [Micro
 * Smartly crop images using [Microsoft Azure's AI Vision API](https://azure.microsoft.com/en-us/services/cognitive-services/computer-vision/)
 * Scan PDF files for embedded text and save for use in post meta using [Microsoft Azure's AI Vision API](https://azure.microsoft.com/en-us/services/cognitive-services/computer-vision/)
 * Bulk classify content with [WP-CLI](https://wp-cli.org/)
+* Modification of your `robots.txt` file to block the most common AI data scraping bots from indexing your site
 
 ### Language Processing
 
@@ -158,7 +159,7 @@ ClassifAI is a sophisticated solution that we want organizations of all shapes a
 - Check for an email from `ClassifAI Team` which contains the registration key.
 - Note that the email will be sent from `opensource@10up.com`, so please whitelist this email address if needed.
 
-### 2. Configure ClassifAI Registration Key under Tools > ClassifAI > ClassifAI Registration
+### 2. Configure ClassifAI Registration Key under Tools > ClassifAI > Settings
 
 - In the `Registered Email` field, enter the email you used for registration.
 - In the `Registration Key` field, enter the registration key from the email in step 1 above.

assets/img/screenshot-6.png

@@ -63,7 +63,7 @@
 
 $services_menu      = Classifai\get_services_menu();
 $classifai_settings = array(
-	'classifai_settings' => __( 'ClassifAI Registration', 'classifai' ),
+	'classifai_settings' => __( 'Settings', 'classifai' ),
 );
 
 $classifai_header_menu = array_merge( $classifai_settings, $services_menu );

includes/Classifai/Admin/templates/classifai-header.php

@@ -39,7 +39,8 @@ public function enable() {
 		add_action( 'admin_init', [ $this, 'add_privacy_policy_content' ] );
 		add_action( 'admin_init', [ $this, 'maybe_migrate_to_v3' ] );
 		add_action( 'admin_enqueue_scripts', [ $this, 'enqueue_admin_assets' ] );
-		add_filter( 'plugin_action_links_' . CLASSIFAI_PLUGIN_BASENAME, array( $this, 'filter_plugin_action_links' ) );
+		add_filter( 'plugin_action_links_' . CLASSIFAI_PLUGIN_BASENAME, [ $this, 'filter_plugin_action_links' ] );
+		add_filter( 'robots_txt', [ $this, 'maybe_block_ai_crawlers' ] );
 		add_action( 'after_classifai_init', [ $this, 'load_action_scheduler' ] );
 	}
 
@@ -248,6 +249,59 @@ public function filter_plugin_action_links( $links ): array {
 		);
 	}
 
+	/**
+	 * Maybe block AI crawlers from indexing the site.
+	 *
+	 * @param string $robots_txt The robots.txt content.
+	 * @return string The robots.txt content.
+	 */
+	public function maybe_block_ai_crawlers( $robots_txt ) {
+		$service_manager = new Services\ServicesManager();
+		$settings        = $service_manager->get_settings();
+
+		// Only block AI bots if the setting is enabled.
+		if ( ! isset( $settings['block_ai_bots'] ) || '1' !== $settings['block_ai_bots'] ) {
+			return $robots_txt;
+		}
+
+		// Ensure the content is a string, in case some other plugin has messed up.
+		if ( ! is_string( $robots_txt ) ) {
+			$robots_txt = (string) $robots_txt;
+		}
+
+		$robots_txt .= '
+## Apple crawler (https://support.apple.com/en-us/119829)
+User-agent: Applebot-Extended
+Disallow: /
+
+## Common Crawl crawler (https://commoncrawl.org/ccbot)
+User-agent: CCBot
+Disallow: /
+
+## Anthropic crawler (https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler)
+User-agent: ClaudeBot
+Disallow: /
+
+## Facebook crawler (https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/)
+User-agent: FacebookBot
+Disallow: /
+
+## Google crawler (https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers)
+User-agent: Google-Extended
+Disallow: /
+
+## OpenAI GPTBot crawler (https://platform.openai.com/docs/bots)
+User-agent: GPTbot
+Disallow: /
+
+## Meta crawler (https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/)
+User-agent: Meta-ExternalAgent
+Disallow: /
+';
+
+		return $robots_txt;
+	}
+
 	/**
 	 * Load the Action Scheduler library.
 	 */

includes/Classifai/Plugin.php

@@ -138,7 +138,7 @@ public function register_recommendation_service_features( array $features ): arr
 	/**
 	 * Get general ClassifAI settings
 	 *
-	 * @param string $index Optional specific setting to be retrieved.
+	 * @param string|bool $index Optional specific setting to be retrieved. If false, all settings will be returned.
 	 * @return mixed
 	 */
 	public function get_settings( $index = false ) {
@@ -187,8 +187,9 @@ public function register_settings() {
 	public function sanitize_settings( $settings ): array {
 		$new_settings = [];
 
-		if ( isset( $settings['email'] )
-			&& isset( $settings['license_key'] )
+		// Save registration settings.
+		if ( ! empty( $settings['email'] )
+			&& ! empty( $settings['license_key'] )
 			&& $this->check_license_key( $settings['email'], $settings['license_key'] )
 		) {
 			$new_settings['valid_license'] = true;
@@ -207,6 +208,9 @@ public function sanitize_settings( $settings ): array {
 			);
 		}
 
+		// Save block AI bots setting.
+		$new_settings['block_ai_bots'] = $settings['block_ai_bots'] ?? '0';
+
 		return $new_settings;
 	}
 

includes/Classifai/Services/ServicesManager.php

@@ -35,6 +35,7 @@ Tap into leading cloud-based services like [OpenAI](https://openai.com/), [Micro
 * Smartly crop images using [Microsoft Azure's AI Vision API](https://azure.microsoft.com/en-us/services/cognitive-services/computer-vision/)
 * Scan PDF files for embedded text and save for use in post meta using [Microsoft Azure's AI Vision API](https://azure.microsoft.com/en-us/services/cognitive-services/computer-vision/)
 * Bulk classify content with [WP-CLI](https://wp-cli.org/)
+* Modification of your `robots.txt` file to block the most common AI data scraping bots from indexing your site
 
 **Requirements**
 

readme.txt

@@ -11,6 +11,7 @@ import {
 	Flex,
 	FlexItem,
 	__experimentalInputControl as InputControl, // eslint-disable-line @wordpress/no-unsafe-wp-apis
+	ToggleControl,
 } from '@wordpress/components';
 import { __, sprintf } from '@wordpress/i18n';
 import apiFetch from '@wordpress/api-fetch';
@@ -105,6 +106,7 @@ export const ClassifAIRegistrationForm = ( { onSaveSuccess = () => {} } ) => {
 							onChange={ ( value ) => {
 								setSettings( { ...settings, email: value } );
 							} }
+							__next40pxDefaultSize
 						/>
 					</SettingsRow>
 					<SettingsRow
@@ -140,6 +142,37 @@ export const ClassifAIRegistrationForm = ( { onSaveSuccess = () => {} } ) => {
 									license_key: value,
 								} );
 							} }
+							__next40pxDefaultSize
+						/>
+					</SettingsRow>
+				</PanelBody>
+			</Panel>
+			<Panel
+				header={ __( 'General Settings', 'classifai' ) }
+				className="settings-panel"
+			>
+				<PanelBody>
+					<SettingsRow
+						label={ __( 'Block AI Bots', 'classifai' ) }
+						description={
+							<>
+								{ __(
+									'If you turn on this setting, ClassifAI will modify your robots.txt file to request that known AI data scraping bots do not index your site. This will not block AI search bots, just data scraping bots. At the moment, the following bots are blocked: Applebot-Extended, CCBot, ClaudeBot, FacebookBot, Google-Extended, GPTbot, Meta-ExternalAgent.',
+									'classifai'
+								) }
+							</>
+						}
+					>
+						<ToggleControl
+							className="classifai-enable-bot-block"
+							checked={ settings?.block_ai_bots === '1' }
+							onChange={ ( value ) => {
+								setSettings( {
+									...settings,
+									block_ai_bots: value ? '1' : '0',
+								} );
+							} }
+							__nextHasNoMarginBottom
 						/>
 					</SettingsRow>
 				</PanelBody>

src/js/settings/components/classifai-registration/index.js

@@ -108,15 +108,15 @@ export const ServiceNavigation = () => {
 					</NavLink>
 				) ) }
 				<NavLink
-					to="classifai_registration"
-					key="classifai_registration"
+					to="settings"
+					key="settings"
 					className={ ( { isActive } ) =>
 						isActive
 							? 'active-tab classifai-tabs-item'
 							: 'classifai-tabs-item'
 					}
 				>
-					{ __( 'ClassifAI Registration', 'classifai' ) }
+					{ __( 'Settings', 'classifai' ) }
 				</NavLink>
 			</div>
 		</>
@@ -239,7 +239,7 @@ export const ClassifAISettings = () => {
 							element={ <FeatureSettingsWrapper /> }
 						/>
 						<Route
-							path="classifai_registration"
+							path="settings"
 							element={ <ClassifAIRegistration /> }
 						/>
 						{ /* When no routes match, it will redirect to this route path. Note that it should be registered above. */ }

src/js/settings/components/classifai-settings/index.js

@@ -47,4 +47,69 @@ describe( 'Admin can login and make sure plugin is activated', () => {
 			.first()
 			.contains( 'Image Processing' );
 	} );
+
+	it( 'Can visit the general settings page and see all settings.', () => {
+		// Check Selected Navigation menu
+		cy.visitFeatureSettings( 'settings' );
+		cy.get( '.classifai-tabs' ).should( 'exist' );
+		cy.get( '.classifai-tabs a.active-tab' )
+			.first()
+			.contains( 'Settings' );
+
+		// Check that all settings are present.
+		cy.get( '.components-input-control input[type="email"]' ).should(
+			'exist'
+		);
+		cy.get( '.components-input-control input[type="password"]' ).should(
+			'exist'
+		);
+		cy.get( '.classifai-enable-bot-block input' ).should( 'exist' );
+		cy.get( '.classifai-enable-bot-block input' ).should(
+			'not.be.checked'
+		);
+	} );
+
+	it( 'Can turn on "Block AI Bots" setting and it works.', () => {
+		cy.visitFeatureSettings( 'settings' );
+
+		cy.get( '.classifai-enable-bot-block input' ).check();
+
+		cy.saveGeneralSettings();
+
+		// Check that the robots.txt file has bots blocked.
+		cy.request( '/robots.txt' ).then( ( response ) => {
+			expect( response.body ).to.contain(
+				'User-agent: Applebot-Extended'
+			);
+			expect( response.body ).to.contain( 'User-agent: CCBot' );
+			expect( response.body ).to.contain( 'User-agent: ClaudeBot' );
+			expect( response.body ).to.contain( 'User-agent: FacebookBot' );
+			expect( response.body ).to.contain( 'User-agent: Google-Extended' );
+			expect( response.body ).to.contain( 'User-agent: GPTbot' );
+			expect( response.body ).to.contain(
+				'User-agent: Meta-ExternalAgent'
+			);
+		} );
+
+		cy.get( '.classifai-enable-bot-block input' ).uncheck();
+
+		cy.saveGeneralSettings();
+
+		// Check that the robots.txt file has bots unblocked.
+		cy.request( '/robots.txt' ).then( ( response ) => {
+			expect( response.body ).to.not.contain(
+				'User-agent: Applebot-Extended'
+			);
+			expect( response.body ).to.not.contain( 'User-agent: CCBot' );
+			expect( response.body ).to.not.contain( 'User-agent: ClaudeBot' );
+			expect( response.body ).to.not.contain( 'User-agent: FacebookBot' );
+			expect( response.body ).to.not.contain(
+				'User-agent: Google-Extended'
+			);
+			expect( response.body ).to.not.contain( 'User-agent: GPTbot' );
+			expect( response.body ).to.not.contain(
+				'User-agent: Meta-ExternalAgent'
+			);
+		} );
+	} );
 } );