ability to load stopwords from a PHP file or a TXT file

Author: michaelbaril

README.md

@@ -315,7 +315,36 @@ return [
 ];
 ```
 
-In the example, the stemmer comes from the package [`wamania/php-stemmer`],
+In the example, the stemmer comes from the package
+[`wamania/php-stemmer`](https://github.com/wamania/php-stemmer),
 but any class with a `stem` method, or anything callable such as a closure, will do.
 
-[`wamania/php-stemmer`]: https://github.com/wamania/php-stemmer
+As for stopwords, you can either list them directly in the config (as shown above),
+or load them from a file. The file can be either a TXT file (with one stopword per line),
+or a PHP file that returns an array:
+
+```php
+// config/scout.php
+return [
+    // ...
+    'sqlout' => [
+        // ...
+        'stopwords' => 'storage/app/stopwords/fr.php',
+        // ...
+    ],
+];
+```
+
+```php
+// storage/app/stopwords/fr.php
+return [
+    'à',
+    'le',
+    'la',
+];
+```
+
+You may want to use the package [`voku/stop-words`](https://github.com/voku/stop-words),
+which provides collections of stopwords for various languages as PHP files,
+or [`yooper/stop-words`](https://github.com/yooper/stop-words),
+which provides them as TXT files.

composer.json

@@ -13,9 +13,12 @@
         "laravel/scout": "^9.0|^10.0"
     },
     "require-dev": {
+        "orchestra/testbench": "^6.23|^7.0|^8.0|^9.0|^10.0",
+        "phpunit/phpunit": "^9.0|^10.0|^11.0",
         "squizlabs/php_codesniffer": "^3.7",
         "wamania/php-stemmer": "^3.0|^4.0",
-        "orchestra/testbench": "^6.23|^7.0|^8.0|^9.0|^10.0"
+        "voku/stop-words": "^2.0",
+        "yooper/stop-words": "^1.0"
     },
     "autoload": {
         "files": [
@@ -31,7 +34,9 @@
         }
     },
     "suggest": {
-        "wamania/php-stemmer": "PHP stemmer that can be used together with Sqlout."
+        "wamania/php-stemmer": "PHP stemmer that can be used together with Sqlout.",
+        "voku/stop-words": "A collection of stop words for various languages.",
+        "yooper/stop-words": "Another collection of stop words."
     },
     "extra": {
         "laravel": {

src/Engine.php

@@ -41,7 +41,7 @@ protected function processString($content)
 
         // Remove stopwords & short words:
         $minLength = config('scout.sqlout.minimum_length', 0);
-        $stopwords = config('scout.sqlout.stopwords', []);
+        $stopwords = $this->loadStopWords();
         $words = (new Collection($words))->reject(function ($word) use ($minLength, $stopwords) {
             return mb_strlen($word) < $minLength || in_array($word, $stopwords);
         })->all();
@@ -69,6 +69,32 @@ protected function processString($content)
         return implode(' ', $words);
     }
 
+    protected function loadStopWords()
+    {
+        $stopwords = config('scout.sqlout.stopwords', []);
+        if (is_iterable($stopwords)) {
+            return $stopwords;
+        }
+
+        $file = $stopwords;
+        if (!file_exists($file)) {
+            throw new Exception("Can't import stop words from $file");
+        }
+
+        $stream = fopen($file, 'r');
+        $firstline = trim(fgets($stream));
+
+        if (trim($firstline) == '<?php') {
+            return require $file;
+        }
+
+        $stopwords = [$firstline];
+        while (false !== ($word = fgets($stream))) {
+            $stopwords[] = trim($word);
+        }
+        return $stopwords;
+    }
+
     /**
      * Update the given model in the index.
      *

tests/SearchTest.php

@@ -242,18 +242,40 @@ public function test_filters()
         $this->assertEquals('salut ça boume ?', $indexed);
     }
 
-    public function test_stopwords()
+    /**
+     * @dataProvider stopWordsProvider
+     */    
+    public function test_stopwords($config, $content, $expectedIndexedContent)
     {
-        app('config')->set('scout.sqlout.stopwords', [
-            'fuck',
-        ]);
+        app('config')->set('scout.sqlout.stopwords', $config);
 
         $post = Post::first();
-        $post->body = 'shut the fuck up donny';
+        $post->body = $content;
         $post->save();
 
         $indexed = $this->newSearchQuery()->where('record_type', Post::class)->where('record_id', $post->id)->where('field', 'body')->value('content');
-        $this->assertEquals('shut the up donny', $indexed);
+        $this->assertEquals($expectedIndexedContent, $indexed);
+    }
+
+    public static function stopWordsProvider()
+    {
+        return [
+            'array' => [
+                ['fuck'],
+                'shut the fuck up donny',
+                'shut the up donny',
+            ],
+            'PHP file' => [
+                'vendor/voku/stop-words/src/voku/helper/stopwords/fr.php',
+                'banco charlie alpha bravo',
+                'charlie alpha bravo',
+            ],
+            'TXT file' => [
+                'vendor/yooper/stop-words/data/stop-words_french_1_fr.txt',
+                'bigre boum tsoin brrr kiki',
+                'kiki',
+            ],
+        ];
     }
 
     public function test_minimum_length()