Immutable settings (#22)

* Make setting immutable between each scrape

* Validate that custom settings is a dictionary

* Change value of custom settings in tests

* lint

* Removed unecessary try and catch

* Update scraper/src/meilisearch_helper.py

Co-authored-by: Clémentine Urquizar <[email protected]>

Co-authored-by: Clémentine Urquizar <[email protected]>

Author: bidoubiwa

scraper/src/config/config_validator.py

@@ -25,6 +25,11 @@ def validate(self):
                                                     list):
             raise Exception('stop_urls should be list')
 
+        # Custom settings must be a dict
+        if self.config.custom_settings and not isinstance(self.config.custom_settings,
+                                                    dict):
+            raise Exception('custom_settings must be a dictionary')
+
         if self.config.js_render and not isinstance(self.config.js_render,
                                                     bool):
             raise Exception('js_render should be boolean')

scraper/src/index.py

@@ -40,6 +40,7 @@ def run_config(config):
         config.app_id,
         config.api_key,
         config.index_uid,
+        config.custom_settings
     )
 
     root_module = 'src.' if __name__ == '__main__' else 'scraper.src.'

scraper/src/meilisearch_helper.py

@@ -101,11 +101,12 @@ class MeiliSearchHelper:
         'acceptNewFields': False
     }
 
-    def __init__(self, host_url, api_key, index_uid):
+    def __init__(self, host_url, api_key, index_uid, custom_settings):
         self.meilisearch_client = meilisearch.Client(host_url, api_key)
         self.__delete_and_create_index(index_uid)
         self.meilisearch_index = self.__delete_and_create_index(index_uid)
-        self.meilisearch_index.update_settings(MeiliSearchHelper.SETTINGS)
+        settings = {**MeiliSearchHelper.SETTINGS, **custom_settings}
+        self.meilisearch_index.update_settings(settings)
 
     def add_records(self, records, url, from_sitemap):
         """Add new records to the index"""

scraper/src/tests/config_loader/abstract.py

@@ -7,7 +7,7 @@ def config(additional_config={}):
         'allowed_domains': 'allowed_domains',
         'api_key': 'api_key',
         'app_id': 'app_id',
-        'custom_settings': 'custom_settings',
+        'custom_settings': {},
         'hash_strategy': 'hash_strategy',
         'index_uid': 'index_uid',
         'selectors': [],