[AI Bundle][Store] Add FilterInterface and filters support to Indexer

Author: OskarStark

.github/workflows/integration-tests.yaml

@@ -96,12 +96,16 @@ jobs:
             - name: Install demo dependencies
               uses: ramsey/composer-install@v3
               with:
+                  composer-options: "--no-scripts"
                   working-directory: demo
 
             - name: Link demo
               working-directory: demo
               run: ../link
 
+            - run: composer run-script auto-scripts --no-interaction
+              working-directory: demo
+
             - name: Run demo tests
               working-directory: demo
               run: vendor/bin/phpunit

demo/config/packages/ai.yaml

@@ -62,6 +62,8 @@ ai:
         blog:
             loader: 'Symfony\AI\Store\Document\Loader\RssFeedLoader'
             source: 'https://feeds.feedburner.com/symfony/blog'
+            filters:
+                - 'app.filter.week_of_symfony'
             transformers:
                 - 'Symfony\AI\Store\Document\Transformer\TextTrimTransformer'
             vectorizer: 'ai.vectorizer.openai'
@@ -83,3 +85,9 @@ services:
 
     Symfony\AI\Store\Document\Loader\RssFeedLoader: ~
     Symfony\AI\Store\Document\Transformer\TextTrimTransformer: ~
+    
+    app.filter.week_of_symfony:
+        class: 'Symfony\AI\Store\Document\Filter\TextContainsFilter'
+        arguments:
+            $needle: 'Week of Symfony'
+            $caseSensitive: false

examples/indexer/index-with-filters.php

@@ -0,0 +1,88 @@
+<?php
+
+/*
+ * This file is part of the Symfony package.
+ *
+ * (c) Fabien Potencier <[email protected]>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+use Symfony\AI\Platform\Bridge\OpenAi\Embeddings;
+use Symfony\AI\Platform\Bridge\OpenAi\PlatformFactory;
+use Symfony\AI\Store\Bridge\Local\InMemoryStore;
+use Symfony\AI\Store\Document\Filter\TextContainsFilter;
+use Symfony\AI\Store\Document\Loader\InMemoryLoader;
+use Symfony\AI\Store\Document\Metadata;
+use Symfony\AI\Store\Document\TextDocument;
+use Symfony\AI\Store\Document\Transformer\TextTrimTransformer;
+use Symfony\AI\Store\Document\Vectorizer;
+use Symfony\AI\Store\Indexer;
+use Symfony\Component\Uid\Uuid;
+
+require_once dirname(__DIR__).'/bootstrap.php';
+
+$platform = PlatformFactory::create(env('OPENAI_API_KEY'), http_client());
+$store = new InMemoryStore();
+$vectorizer = new Vectorizer($platform, new Embeddings('text-embedding-3-small'));
+
+// Sample documents with some unwanted content
+$documents = [
+    new TextDocument(
+        Uuid::v4(),
+        'Artificial Intelligence is transforming the way we work and live. Machine learning algorithms can now process vast amounts of data and make predictions with remarkable accuracy.',
+        new Metadata(['title' => 'AI Revolution', 'category' => 'technology'])
+    ),
+    new TextDocument(
+        Uuid::v4(),
+        'Week of Symfony - This week we released several new features including improved performance and better documentation.',
+        new Metadata(['title' => 'Weekly Newsletter', 'category' => 'newsletter'])
+    ),
+    new TextDocument(
+        Uuid::v4(),
+        'SPAM: Buy cheap products now! Limited time offer on all electronics. Click here to save 90% on your purchase!',
+        new Metadata(['title' => 'Advertisement', 'category' => 'spam'])
+    ),
+    new TextDocument(
+        Uuid::v4(),
+        'Climate change is one of the most pressing challenges of our time. Renewable energy sources like solar and wind power are becoming increasingly important for a sustainable future.',
+        new Metadata(['title' => 'Climate Action', 'category' => 'environment'])
+    ),
+];
+
+// Create filters to remove unwanted content
+$filters = [
+    new TextContainsFilter('Week of Symfony', caseSensitive: false),
+    new TextContainsFilter('SPAM:', caseSensitive: true),
+];
+
+$indexer = new Indexer(
+    loader: new InMemoryLoader($documents),
+    vectorizer: $vectorizer,
+    store: $store,
+    source: null,
+    filters: $filters,
+    transformers: [
+        new TextTrimTransformer(),
+    ],
+);
+
+$indexer->index();
+
+$vector = $vectorizer->vectorize('technology artificial intelligence');
+$results = $store->query($vector);
+
+foreach ($results as $i => $document) {
+    $title = $document->metadata['title'] ?? 'Unknown';
+    $category = $document->metadata['category'] ?? 'Unknown';
+    echo sprintf("%d. %s [%s]\n", $i + 1, $title, $category);
+    echo sprintf("   Content: %s\n", substr($document->metadata->getText() ?? 'No content', 0, 80).'...');
+    echo sprintf("   ID: %s\n\n", substr($document->id, 0, 8).'...');
+}
+
+echo "=== Results Summary ===\n";
+echo sprintf("Original documents: %d\n", count($documents));
+echo sprintf("Documents after filtering: %d\n", count($results));
+echo sprintf("Filtered out: %d documents\n", count($documents) - count($results));
+echo "\nThe 'Week of Symfony' newsletter and SPAM advertisement were successfully filtered out!\n";

src/ai-bundle/config/options.php

@@ -557,7 +557,12 @@
                             ->scalarPrototype()->end()
                             ->defaultValue([])
                         ->end()
-                        ->stringNode('vectorizer')
+                        ->arrayNode('filters')
+                            ->info('Array of filter service names')
+                            ->scalarPrototype()->end()
+                            ->defaultValue([])
+                        ->end()
+                        ->scalarNode('vectorizer')
                             ->info('Service name of vectorizer')
                             ->defaultValue(VectorizerInterface::class)
                         ->end()

src/ai-bundle/src/AiBundle.php

@@ -1130,15 +1130,21 @@ private function processVectorizerConfig(string $name, array $config, ContainerB
     private function processIndexerConfig(int|string $name, array $config, ContainerBuilder $container): void
     {
         $transformers = [];
-        foreach ($config['transformers'] ?? [] as $transformer) {
+        foreach ($config['transformers'] as $transformer) {
             $transformers[] = new Reference($transformer);
         }
 
+        $filters = [];
+        foreach ($config['filters'] as $filter) {
+            $filters[] = new Reference($filter);
+        }
+
         $definition = new Definition(Indexer::class, [
             new Reference($config['loader']),
             new Reference($config['vectorizer']),
             new Reference($config['store']),
             $config['source'],
+            $filters,
             $transformers,
             new Reference('logger', ContainerInterface::IGNORE_ON_INVALID_REFERENCE),
         ]);