From 1f3bf6bb3a5429d771acbce5222a91af458b149c Mon Sep 17 00:00:00 2001 From: BrocksiNet Date: Fri, 31 Oct 2025 10:16:28 +0100 Subject: [PATCH 01/10] feat(robots.txt): add guide for extending robots.txt functionality * Introduce a new guide on how to extend robots.txt in Shopware, including event handling for custom validation and dynamic directives. * Update the SEO index to reference the new robots.txt guide. --- .../plugins/content/seo/extend-robots-txt.md | 205 ++++++++++++++++++ guides/plugins/plugins/content/seo/index.md | 14 +- 2 files changed, 217 insertions(+), 2 deletions(-) create mode 100644 guides/plugins/plugins/content/seo/extend-robots-txt.md diff --git a/guides/plugins/plugins/content/seo/extend-robots-txt.md b/guides/plugins/plugins/content/seo/extend-robots-txt.md new file mode 100644 index 000000000..bc74b933d --- /dev/null +++ b/guides/plugins/plugins/content/seo/extend-robots-txt.md @@ -0,0 +1,205 @@ +--- +nav: + title: Extend robots.txt + position: 20 + +--- + +# Extend robots.txt + +## Overview + +Since Shopware 6.7.1, the platform provides full robots.txt support with all standard directives and user-agent blocks. This feature was developed during [Hacktoberfest 2024](https://www.shopware.com/en/news/hacktoberfest-2024-outcome-a-robots-txt-for-shopware/). Learn more about [configuring robots.txt](https://docs.shopware.com/en/shopware-6-en/tutorials-and-faq/creation-of-robots-txt) in the user documentation. + +Starting with Shopware 6.7.4, you can extend the robots.txt functionality through events to: + +* Add custom validation rules during parsing +* Modify or generate directives dynamically +* Support custom or vendor-specific directives +* Prevent warnings for known non-standard directives + +::: info +The event system described in this guide requires Shopware 6.7.4 or later. +::: + +## Prerequisites + +This guide requires you to have a basic plugin running. If you don't know how to create a plugin, head over to the plugin base guide: + + + +You should also be familiar with [Event subscribers](../../plugin-fundamentals/listening-to-events). + +## Modifying parsed directives + +The `RobotsDirectiveParsingEvent` is dispatched after robots.txt content is parsed. You can modify the parsed result, add validation, or inject dynamic directives. + +This example shows how to add AI crawler restrictions and validate crawl-delay values: + + + + +```php + 'onRobotsParsing', + ]; + } + + public function onRobotsParsing(RobotsDirectiveParsingEvent $event): void + { + $parsedRobots = $event->getParsedRobots(); + + // 1. Add restrictions for AI crawlers + $aiCrawlers = ['GPTBot', 'ChatGPT-User', 'CCBot', 'anthropic-ai']; + + $aiBlock = new RobotsUserAgentBlock( + userAgents: $aiCrawlers, + directives: [ + new RobotsDirective( + type: RobotsDirectiveType::DISALLOW, + value: '/checkout/', + ), + ], + ); + + $parsedRobots->addUserAgentBlock($aiBlock); + + // 2. Validate existing crawl-delay values + foreach ($parsedRobots->getUserAgentBlocks() as $block) { + foreach ($block->getDirectives() as $directive) { + if ($directive->getType() === RobotsDirectiveType::CRAWL_DELAY) { + $value = (int) $directive->getValue(); + + if ($value > 60) { + $event->addIssue(new ParseIssue( + severity: ParseIssueSeverity::WARNING, + message: sprintf( + 'Crawl-delay of %d seconds may be too high', + $value + ), + lineNumber: null, + )); + } + } + } + } + + $this->logger->info('Extended robots.txt with AI crawler rules'); + } +} +``` + + + + + +```xml + + + + + + + + + + +``` + + + + +## Handling custom directives + +The `RobotsUnknownDirectiveEvent` is dispatched when an unknown directive is encountered. Use this to support vendor-specific directives or prevent warnings for known non-standard directives: + +```php + 'handleCustomDirective', + ]; + } + + public function handleCustomDirective(RobotsUnknownDirectiveEvent $event): void + { + // Support Google and Yandex specific directives + $knownCustomDirectives = ['noimageindex', 'noarchive', 'clean-param']; + + if (in_array(strtolower($event->getDirectiveName()), $knownCustomDirectives, true)) { + $event->setHandled(true); // Prevent "unknown directive" warning + } + } +} +``` + +Register the subscriber in your `services.xml`: + +```xml + + + +``` + +## Parse issues + +You can add validation warnings or errors during parsing using the `ParseIssue` class: + +```php +use Shopware\Storefront\Page\Robots\Parser\ParseIssue; +use Shopware\Storefront\Page\Robots\Parser\ParseIssueSeverity; + +// Add a warning +$event->addIssue(new ParseIssue( + severity: ParseIssueSeverity::WARNING, + message: 'Consider adding a sitemap directive for better SEO', + lineNumber: null, +)); + +// Add an error +$event->addIssue(new ParseIssue( + severity: ParseIssueSeverity::ERROR, + message: 'Invalid crawl-delay value: must be a positive integer', + lineNumber: 42, +)); +``` + +Issues are automatically logged when the robots.txt configuration is saved in the Administration. Use `WARNING` for recommendations and `ERROR` for critical problems that prevent proper generation. + diff --git a/guides/plugins/plugins/content/seo/index.md b/guides/plugins/plugins/content/seo/index.md index a3a18cf29..c9b6b7843 100644 --- a/guides/plugins/plugins/content/seo/index.md +++ b/guides/plugins/plugins/content/seo/index.md @@ -7,6 +7,16 @@ nav: # SEO -The Shopware SEO feature offers the ability to add custom SEO URLs to optimize the search engine visibility of the e-commerce platform. +The Shopware SEO feature offers comprehensive tools to optimize the search engine visibility of your e-commerce platform. -With the SEO plugin, you can create custom SEO URLs for product pages, categories, content pages, and other relevant sections of the website. The plugin allows businesses to customize meta tags for each page, including meta titles, descriptions, and keywords. This enables users to optimize the on-page SEO elements, providing search engines with relevant information about the content of the page. +## SEO URLs + +You can create custom SEO URLs for product pages, categories, content pages, and other relevant sections of the website. The plugin allows businesses to customize meta tags for each page, including meta titles, descriptions, and keywords. This enables users to optimize the on-page SEO elements, providing search engines with relevant information about the content of the page. + + + +## robots.txt + +Shopware provides full support for robots.txt configuration, including all standard directives, user-agent blocks, and extensibility through events. You can customize how search engine crawlers interact with your shop by extending the robots.txt parsing and generation. + + From f6d8a8e0465f5235662dfa40c160686534cd7b28 Mon Sep 17 00:00:00 2001 From: BrocksiNet Date: Fri, 31 Oct 2025 10:19:34 +0100 Subject: [PATCH 02/10] fix(robots.txt): remove trailing whitespace in robots.txt guide --- guides/plugins/plugins/content/seo/extend-robots-txt.md | 1 - 1 file changed, 1 deletion(-) diff --git a/guides/plugins/plugins/content/seo/extend-robots-txt.md b/guides/plugins/plugins/content/seo/extend-robots-txt.md index bc74b933d..64f9b3e3b 100644 --- a/guides/plugins/plugins/content/seo/extend-robots-txt.md +++ b/guides/plugins/plugins/content/seo/extend-robots-txt.md @@ -202,4 +202,3 @@ $event->addIssue(new ParseIssue( ``` Issues are automatically logged when the robots.txt configuration is saved in the Administration. Use `WARNING` for recommendations and `ERROR` for critical problems that prevent proper generation. - From 9ea6ff233b23dbf501a5a90f54241b2629dbcae8 Mon Sep 17 00:00:00 2001 From: BrocksiNet Date: Fri, 31 Oct 2025 10:29:16 +0100 Subject: [PATCH 03/10] refactor(robots.txt): enhance clarity and consistency in robots.txt documentation * Update titles and references to 'robots.txt file' for consistency. * Improve phrasing and formatting in the overview and directive sections for better readability. * Ensure proper inline code formatting for 'robots.txt' throughout the documentation. --- .../plugins/plugins/content/seo/extend-robots-txt.md | 12 ++++++------ guides/plugins/plugins/content/seo/index.md | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/guides/plugins/plugins/content/seo/extend-robots-txt.md b/guides/plugins/plugins/content/seo/extend-robots-txt.md index 64f9b3e3b..056ca6eef 100644 --- a/guides/plugins/plugins/content/seo/extend-robots-txt.md +++ b/guides/plugins/plugins/content/seo/extend-robots-txt.md @@ -1,17 +1,17 @@ --- nav: - title: Extend robots.txt + title: Extend robots.txt file position: 20 --- -# Extend robots.txt +# Extend robots.txt file ## Overview -Since Shopware 6.7.1, the platform provides full robots.txt support with all standard directives and user-agent blocks. This feature was developed during [Hacktoberfest 2024](https://www.shopware.com/en/news/hacktoberfest-2024-outcome-a-robots-txt-for-shopware/). Learn more about [configuring robots.txt](https://docs.shopware.com/en/shopware-6-en/tutorials-and-faq/creation-of-robots-txt) in the user documentation. +Since Shopware 6.7.1, the platform provides full `robots.txt` support with all standard directives and user-agent blocks. This feature was developed as a community contribution during the 2024 Hacktoberfest event ([learn more](https://www.shopware.com/en/news/hacktoberfest-2024-outcome-a-robots-txt-for-shopware/)). For general configuration, refer to the [user documentation](https://docs.shopware.com/en/shopware-6-en/tutorials-and-faq/creation-of-robots-txt). -Starting with Shopware 6.7.4, you can extend the robots.txt functionality through events to: +Starting with Shopware 6.7.4, you can extend the `robots.txt` functionality through events to: * Add custom validation rules during parsing * Modify or generate directives dynamically @@ -32,7 +32,7 @@ You should also be familiar with [Event subscribers](../../plugin-fundamentals/l ## Modifying parsed directives -The `RobotsDirectiveParsingEvent` is dispatched after robots.txt content is parsed. You can modify the parsed result, add validation, or inject dynamic directives. +The `RobotsDirectiveParsingEvent` is dispatched after `robots.txt` content is parsed. You can modify the parsed result, add validation, or inject dynamic directives. This example shows how to add AI crawler restrictions and validate crawl-delay values: @@ -201,4 +201,4 @@ $event->addIssue(new ParseIssue( )); ``` -Issues are automatically logged when the robots.txt configuration is saved in the Administration. Use `WARNING` for recommendations and `ERROR` for critical problems that prevent proper generation. +Issues are automatically logged when the `robots.txt` configuration is saved in the Administration. Use `WARNING` for recommendations and `ERROR` for critical problems that prevent proper generation. diff --git a/guides/plugins/plugins/content/seo/index.md b/guides/plugins/plugins/content/seo/index.md index c9b6b7843..5e408b68e 100644 --- a/guides/plugins/plugins/content/seo/index.md +++ b/guides/plugins/plugins/content/seo/index.md @@ -15,8 +15,8 @@ You can create custom SEO URLs for product pages, categories, content pages, and -## robots.txt +## robots.txt file -Shopware provides full support for robots.txt configuration, including all standard directives, user-agent blocks, and extensibility through events. You can customize how search engine crawlers interact with your shop by extending the robots.txt parsing and generation. +Shopware provides full support for `robots.txt` configuration, including all standard directives, user-agent blocks, and extensibility through events. You can customize how search engine crawlers interact with your shop by extending the `robots.txt` parsing and generation. - + From f559a4df4d2c3f0b8f36ac2de6ebdf376a6b8917 Mon Sep 17 00:00:00 2001 From: BrocksiNet Date: Fri, 31 Oct 2025 10:46:49 +0100 Subject: [PATCH 04/10] refactor(robots): update terminology and enhance documentation clarity * Change title references from 'robots.txt file' to 'robots configuration' for consistency. * Update version references from 6.7.4 to 6.7.5 in the guide. * Add a new example subscriber for validating sitemap directives and crawl-delay values. * Improve overall structure and readability of the robots configuration documentation. --- .../plugins/content/seo/extend-robots-txt.md | 106 ++++++++++++++---- guides/plugins/plugins/content/seo/index.md | 4 +- 2 files changed, 89 insertions(+), 21 deletions(-) diff --git a/guides/plugins/plugins/content/seo/extend-robots-txt.md b/guides/plugins/plugins/content/seo/extend-robots-txt.md index 056ca6eef..6b4fc9f6d 100644 --- a/guides/plugins/plugins/content/seo/extend-robots-txt.md +++ b/guides/plugins/plugins/content/seo/extend-robots-txt.md @@ -1,17 +1,17 @@ --- nav: - title: Extend robots.txt file + title: Extend robots configuration position: 20 --- -# Extend robots.txt file +# Extend robots configuration ## Overview -Since Shopware 6.7.1, the platform provides full `robots.txt` support with all standard directives and user-agent blocks. This feature was developed as a community contribution during the 2024 Hacktoberfest event ([learn more](https://www.shopware.com/en/news/hacktoberfest-2024-outcome-a-robots-txt-for-shopware/)). For general configuration, refer to the [user documentation](https://docs.shopware.com/en/shopware-6-en/tutorials-and-faq/creation-of-robots-txt). +Since Shopware 6.7.1, the platform provides full `robots.txt` support with all standard directives and user-agent blocks. This feature was developed as an open-source contribution during October 2024 ([learn more](https://www.shopware.com/en/news/hacktoberfest-2024-outcome-a-robots-txt-for-shopware/)). For general configuration, refer to the [user documentation](https://docs.shopware.com/en/shopware-6-en/tutorials-and-faq/creation-of-robots-txt). -Starting with Shopware 6.7.4, you can extend the `robots.txt` functionality through events to: +Starting with Shopware 6.7.5, you can extend the `robots.txt` functionality through events to: * Add custom validation rules during parsing * Modify or generate directives dynamically @@ -19,7 +19,7 @@ Starting with Shopware 6.7.4, you can extend the `robots.txt` functionality thro * Prevent warnings for known non-standard directives ::: info -The event system described in this guide requires Shopware 6.7.4 or later. +The events described in this guide require Shopware 6.7.5 or later. ::: ## Prerequisites @@ -180,25 +180,93 @@ Register the subscriber in your `services.xml`: ## Parse issues -You can add validation warnings or errors during parsing using the `ParseIssue` class: +You can add validation warnings or errors during parsing using the `ParseIssue` class. This example shows a complete subscriber that validates sitemap directives: + + + ```php + 'validateRobots', + ]; + } + + public function validateRobots(RobotsDirectiveParsingEvent $event): void + { + $parsedRobots = $event->getParsedRobots(); + + // Check if sitemap directive exists + $hasSitemap = false; + foreach ($parsedRobots->getDirectives() as $directive) { + if ($directive->getType() === RobotsDirectiveType::SITEMAP) { + $hasSitemap = true; + break; + } + } + + if (!$hasSitemap) { + $event->addIssue(new ParseIssue( + severity: ParseIssueSeverity::WARNING, + message: 'Consider adding a sitemap directive for better SEO', + lineNumber: null, + )); + } + + // Validate crawl-delay values + foreach ($parsedRobots->getUserAgentBlocks() as $block) { + foreach ($block->getDirectives() as $directive) { + if ($directive->getType() === RobotsDirectiveType::CRAWL_DELAY) { + $value = (int) $directive->getValue(); + + if ($value <= 0) { + $event->addIssue(new ParseIssue( + severity: ParseIssueSeverity::ERROR, + message: 'Invalid crawl-delay value: must be a positive integer', + lineNumber: null, + )); + } + } + } + } + } +} +``` + + + + + +```xml + + -// Add a warning -$event->addIssue(new ParseIssue( - severity: ParseIssueSeverity::WARNING, - message: 'Consider adding a sitemap directive for better SEO', - lineNumber: null, -)); - -// Add an error -$event->addIssue(new ParseIssue( - severity: ParseIssueSeverity::ERROR, - message: 'Invalid crawl-delay value: must be a positive integer', - lineNumber: 42, -)); + + + + + + ``` + + + Issues are automatically logged when the `robots.txt` configuration is saved in the Administration. Use `WARNING` for recommendations and `ERROR` for critical problems that prevent proper generation. diff --git a/guides/plugins/plugins/content/seo/index.md b/guides/plugins/plugins/content/seo/index.md index 5e408b68e..2bc55d434 100644 --- a/guides/plugins/plugins/content/seo/index.md +++ b/guides/plugins/plugins/content/seo/index.md @@ -15,8 +15,8 @@ You can create custom SEO URLs for product pages, categories, content pages, and -## robots.txt file +## Robots configuration Shopware provides full support for `robots.txt` configuration, including all standard directives, user-agent blocks, and extensibility through events. You can customize how search engine crawlers interact with your shop by extending the `robots.txt` parsing and generation. - + From 64a7e428c4cd3ced6bf2cf9574dae6666a733fff Mon Sep 17 00:00:00 2001 From: BrocksiNet Date: Fri, 31 Oct 2025 11:50:01 +0100 Subject: [PATCH 05/10] docs(robots): enhance robots.txt extension guide with new examples * Add tabs for better organization of code examples related to custom directive subscribers and service registration. * Include a complete XML service definition for the CustomDirectiveSubscriber. * Improve overall structure and readability of the documentation. --- .../plugins/content/seo/extend-robots-txt.md | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/guides/plugins/plugins/content/seo/extend-robots-txt.md b/guides/plugins/plugins/content/seo/extend-robots-txt.md index 6b4fc9f6d..b4b466f44 100644 --- a/guides/plugins/plugins/content/seo/extend-robots-txt.md +++ b/guides/plugins/plugins/content/seo/extend-robots-txt.md @@ -139,6 +139,9 @@ class RobotsExtensionSubscriber implements EventSubscriberInterface The `RobotsUnknownDirectiveEvent` is dispatched when an unknown directive is encountered. Use this to support vendor-specific directives or prevent warnings for known non-standard directives: + + + ```php + + ```xml - - - + + + + + + + + + ``` + + + ## Parse issues You can add validation warnings or errors during parsing using the `ParseIssue` class. This example shows a complete subscriber that validates sitemap directives: From daa5d3aefa0407de1bd1ca55fdb6d82452256079 Mon Sep 17 00:00:00 2001 From: BrocksiNet Date: Mon, 3 Nov 2025 14:56:03 +0100 Subject: [PATCH 06/10] docs(robots.txt): update title for clarity in robots.txt extension guide * Change title from 'Extend robots configuration' to 'Extend robots.txt configuration' for improved specificity and consistency. --- guides/plugins/plugins/content/seo/extend-robots-txt.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/guides/plugins/plugins/content/seo/extend-robots-txt.md b/guides/plugins/plugins/content/seo/extend-robots-txt.md index b4b466f44..21a74ac2f 100644 --- a/guides/plugins/plugins/content/seo/extend-robots-txt.md +++ b/guides/plugins/plugins/content/seo/extend-robots-txt.md @@ -1,11 +1,11 @@ --- nav: - title: Extend robots configuration + title: Extend robots.txt configuration position: 20 --- -# Extend robots configuration +# Extend robots.txt configuration ## Overview From b0003bb4e30cdd7dae7dff4caaa2c260573125d4 Mon Sep 17 00:00:00 2001 From: Micha Date: Mon, 3 Nov 2025 15:27:33 +0100 Subject: [PATCH 07/10] chore/fix-spellcheck --- .wordlist.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/.wordlist.txt b/.wordlist.txt index 5306e83e1..640873994 100644 --- a/.wordlist.txt +++ b/.wordlist.txt @@ -1852,6 +1852,7 @@ triggerDeprecated truthy tsx twigjs +txt typeAndCheck typeAndCheckSearchField typeLegacySelectAndCheck From 60d3f3741a9d8586338834caaea5730e0ffb426b Mon Sep 17 00:00:00 2001 From: BrocksiNet Date: Mon, 3 Nov 2025 19:48:59 +0100 Subject: [PATCH 08/10] docs(robots.txt): standardize code block syntax in robots.txt extension guide * Change code block syntax from lowercase to uppercase for PHP and XML examples for consistency and improved readability. * Ensure uniformity in formatting across all code examples in the documentation. --- .../plugins/plugins/content/seo/extend-robots-txt.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/guides/plugins/plugins/content/seo/extend-robots-txt.md b/guides/plugins/plugins/content/seo/extend-robots-txt.md index 21a74ac2f..5e80b624c 100644 --- a/guides/plugins/plugins/content/seo/extend-robots-txt.md +++ b/guides/plugins/plugins/content/seo/extend-robots-txt.md @@ -39,7 +39,7 @@ This example shows how to add AI crawler restrictions and validate crawl-delay v -```php +```PHP -```xml +```XML -```php +```PHP -```xml +```XML -```php +```PHP -```xml +```XML Date: Tue, 4 Nov 2025 08:47:17 +0100 Subject: [PATCH 09/10] docs(robots.txt): update event terminology and enhance examples in robots.txt extension guide * Replace references to 'EventSubscriber' with 'EventListener' for consistency in terminology. * Update code examples to reflect the new naming conventions and improve clarity. * Add validation checks for conflicting Allow/Disallow directives in the robots.txt parsing process. * Enhance overall structure and readability of the documentation. --- .wordlist.txt | 3 + .../plugins/content/seo/extend-robots-txt.md | 160 ++++++++---------- .../data-handling/add-data-translations.md | 1 + 3 files changed, 72 insertions(+), 92 deletions(-) diff --git a/.wordlist.txt b/.wordlist.txt index 640873994..542b20f04 100644 --- a/.wordlist.txt +++ b/.wordlist.txt @@ -355,6 +355,8 @@ Enums EqualsAny ErrorsFacade EventListener +EventListeners +EventSubscriber EventSubscriberInterface Everytime ExampleAppWithCookies @@ -424,6 +426,7 @@ GuestWishlistPage GuestWishlistPagelet HMAC HMR +Hacktober HeaderPagelet Homebrew Hono diff --git a/guides/plugins/plugins/content/seo/extend-robots-txt.md b/guides/plugins/plugins/content/seo/extend-robots-txt.md index 5e80b624c..495f47fe6 100644 --- a/guides/plugins/plugins/content/seo/extend-robots-txt.md +++ b/guides/plugins/plugins/content/seo/extend-robots-txt.md @@ -9,71 +9,66 @@ nav: ## Overview -Since Shopware 6.7.1, the platform provides full `robots.txt` support with all standard directives and user-agent blocks. This feature was developed as an open-source contribution during October 2024 ([learn more](https://www.shopware.com/en/news/hacktoberfest-2024-outcome-a-robots-txt-for-shopware/)). For general configuration, refer to the [user documentation](https://docs.shopware.com/en/shopware-6-en/tutorials-and-faq/creation-of-robots-txt). +Since Shopware 6.7.1, the platform provides full `robots.txt` support with all standard directives and user-agent blocks. +This feature was developed as an open-source contribution during Hacktober 2024 ([learn more](https://www.shopware.com/en/news/hacktoberfest-2024-outcome-a-robots-txt-for-shopware/)). +For general configuration, refer to the [user documentation](https://docs.shopware.com/en/shopware-6-en/tutorials-and-faq/creation-of-robots-txt). -Starting with Shopware 6.7.5, you can extend the `robots.txt` functionality through events to: +::: info +The events and features described in this guide are available since Shopware 6.7.5. +::: +You can extend the `robots.txt` functionality through events to: * Add custom validation rules during parsing * Modify or generate directives dynamically * Support custom or vendor-specific directives * Prevent warnings for known non-standard directives -::: info -The events described in this guide require Shopware 6.7.5 or later. -::: - ## Prerequisites This guide requires you to have a basic plugin running. If you don't know how to create a plugin, head over to the plugin base guide: -You should also be familiar with [Event subscribers](../../plugin-fundamentals/listening-to-events). +You should also be familiar with [Event listeners](../../plugin-fundamentals/listening-to-events). + +::: info +This guide uses EventListeners since each example listens to a single event. If you need to subscribe to multiple events in the same class, consider using an [EventSubscriber](../../plugin-fundamentals/listening-to-events#listening-to-events-via-subscriber) instead. +::: ## Modifying parsed directives The `RobotsDirectiveParsingEvent` is dispatched after `robots.txt` content is parsed. You can modify the parsed result, add validation, or inject dynamic directives. -This example shows how to add AI crawler restrictions and validate crawl-delay values: +This example shows how to dynamically add restrictions for AI crawlers: - + ```PHP 'onRobotsParsing', - ]; - } - - public function onRobotsParsing(RobotsDirectiveParsingEvent $event): void + public function __invoke(RobotsDirectiveParsingEvent $event): void { $parsedRobots = $event->getParsedRobots(); - // 1. Add restrictions for AI crawlers + // Add restrictions for AI crawlers $aiCrawlers = ['GPTBot', 'ChatGPT-User', 'CCBot', 'anthropic-ai']; $aiBlock = new RobotsUserAgentBlock( @@ -88,26 +83,6 @@ class RobotsExtensionSubscriber implements EventSubscriberInterface $parsedRobots->addUserAgentBlock($aiBlock); - // 2. Validate existing crawl-delay values - foreach ($parsedRobots->getUserAgentBlocks() as $block) { - foreach ($block->getDirectives() as $directive) { - if ($directive->getType() === RobotsDirectiveType::CRAWL_DELAY) { - $value = (int) $directive->getValue(); - - if ($value > 60) { - $event->addIssue(new ParseIssue( - severity: ParseIssueSeverity::WARNING, - message: sprintf( - 'Crawl-delay of %d seconds may be too high', - $value - ), - lineNumber: null, - )); - } - } - } - } - $this->logger->info('Extended robots.txt with AI crawler rules'); } } @@ -124,9 +99,9 @@ class RobotsExtensionSubscriber implements EventSubscriberInterface xsi:schemaLocation="http://symfony.com/schema/dic/services http://symfony.com/schema/dic/services/services-1.0.xsd"> - + - + @@ -140,28 +115,20 @@ class RobotsExtensionSubscriber implements EventSubscriberInterface The `RobotsUnknownDirectiveEvent` is dispatched when an unknown directive is encountered. Use this to support vendor-specific directives or prevent warnings for known non-standard directives: - + ```PHP 'handleCustomDirective', - ]; - } - - public function handleCustomDirective(RobotsUnknownDirectiveEvent $event): void + public function __invoke(RobotsUnknownDirectiveEvent $event): void { // Support Google and Yandex specific directives $knownCustomDirectives = ['noimageindex', 'noarchive', 'clean-param']; @@ -184,8 +151,8 @@ class CustomDirectiveSubscriber implements EventSubscriberInterface xsi:schemaLocation="http://symfony.com/schema/dic/services http://symfony.com/schema/dic/services/services-1.0.xsd"> - - + + @@ -194,56 +161,31 @@ class CustomDirectiveSubscriber implements EventSubscriberInterface -## Parse issues +## Validation and parse issues -You can add validation warnings or errors during parsing using the `ParseIssue` class. This example shows a complete subscriber that validates sitemap directives: +You can add validation warnings or errors during parsing using the `ParseIssue` class. This example shows common validation scenarios: - + ```PHP 'validateRobots', - ]; - } - - public function validateRobots(RobotsDirectiveParsingEvent $event): void + public function __invoke(RobotsDirectiveParsingEvent $event): void { $parsedRobots = $event->getParsedRobots(); - // Check if sitemap directive exists - $hasSitemap = false; - foreach ($parsedRobots->getDirectives() as $directive) { - if ($directive->getType() === RobotsDirectiveType::SITEMAP) { - $hasSitemap = true; - break; - } - } - - if (!$hasSitemap) { - $event->addIssue(new ParseIssue( - severity: ParseIssueSeverity::WARNING, - message: 'Consider adding a sitemap directive for better SEO', - lineNumber: null, - )); - } - // Validate crawl-delay values foreach ($parsedRobots->getUserAgentBlocks() as $block) { foreach ($block->getDirectives() as $directive) { @@ -257,6 +199,40 @@ class RobotsValidationSubscriber implements EventSubscriberInterface lineNumber: null, )); } + + if ($value > 10) { + $event->addIssue(new ParseIssue( + severity: ParseIssueSeverity::WARNING, + message: 'Crawl-delay value is very high. This may significantly slow down indexing.', + lineNumber: null, + )); + } + } + } + } + + // Check for conflicting Allow/Disallow directives + foreach ($parsedRobots->getUserAgentBlocks() as $block) { + $disallowed = []; + $allowed = []; + + foreach ($block->getDirectives() as $directive) { + if ($directive->getType() === RobotsDirectiveType::DISALLOW) { + $disallowed[] = $directive->getValue(); + } elseif ($directive->getType() === RobotsDirectiveType::ALLOW) { + $allowed[] = $directive->getValue(); + } + } + + foreach ($allowed as $allowPath) { + foreach ($disallowed as $disallowPath) { + if ($allowPath === $disallowPath) { + $event->addIssue(new ParseIssue( + severity: ParseIssueSeverity::WARNING, + message: sprintf('Conflicting directives: Path "%s" is both allowed and disallowed', $allowPath), + lineNumber: null, + )); + } } } } @@ -275,8 +251,8 @@ class RobotsValidationSubscriber implements EventSubscriberInterface xsi:schemaLocation="http://symfony.com/schema/dic/services http://symfony.com/schema/dic/services/services-1.0.xsd"> - - + + diff --git a/guides/plugins/plugins/framework/data-handling/add-data-translations.md b/guides/plugins/plugins/framework/data-handling/add-data-translations.md index f69e1daf3..73ada1fd6 100644 --- a/guides/plugins/plugins/framework/data-handling/add-data-translations.md +++ b/guides/plugins/plugins/framework/data-handling/add-data-translations.md @@ -297,3 +297,4 @@ class ExampleDefinition extends EntityDefinition ]); } } +``` From 9096121d6eae6b468c8a67d57712006b50179612 Mon Sep 17 00:00:00 2001 From: BrocksiNet Date: Tue, 4 Nov 2025 10:23:38 +0100 Subject: [PATCH 10/10] docs(robots.txt): fix markdown lint error --- guides/plugins/plugins/content/seo/extend-robots-txt.md | 1 + 1 file changed, 1 insertion(+) diff --git a/guides/plugins/plugins/content/seo/extend-robots-txt.md b/guides/plugins/plugins/content/seo/extend-robots-txt.md index 495f47fe6..83abac59e 100644 --- a/guides/plugins/plugins/content/seo/extend-robots-txt.md +++ b/guides/plugins/plugins/content/seo/extend-robots-txt.md @@ -18,6 +18,7 @@ The events and features described in this guide are available since Shopware 6.7 ::: You can extend the `robots.txt` functionality through events to: + * Add custom validation rules during parsing * Modify or generate directives dynamically * Support custom or vendor-specific directives