Add a locking mechanism when accessing CSV files.

This patch touches #435.

Indeed, as discovered by @Yuki2718 and @DandelionSprout, our processes sometimes
trigger a PermissionError exception on Windows.

While testing with one of the latest builds of Windows 11 (Pro), I was able to
reproduce the issue sometimes and therefore analyze it in more detail. It turns out
that in some rare cases, 2 processes are simultaneously attempting to read the same
CSV file.

This patch attempts to solve the problem by introducing a shared lock which - for now
- only affect Windows-Users. When running on Windows, all processes will share a lock
that blocks when 2 processes try to access the same ressource at the same time.

Contributors:
  * @Yuki2718
  * @DandelionSprout

Author: funilrys

PyFunceble/checker/availability/base.py

@@ -54,7 +54,7 @@
 
 import multiprocessing
 from datetime import datetime
-from typing import Dict, List, Optional
+from typing import Any, Dict, List, Optional
 
 from sqlalchemy.orm import Session
 
@@ -110,8 +110,12 @@ class AvailabilityCheckerBase(CheckerBase):
         Optional, Activates/Disables the check of the status before the actual
         status gathering.
     :param bool use_whois_db:
-        Optional, Activates/Disable the usage of a local database to store the
+        Optional, Activates/Disables the usage of a local database to store the
         WHOIS datasets.
+    :param bool use_platform:
+        Optional, Activates/Disables the usage of the platform features.
+    :param multiprocessing.Lock shared_lock:
+        Optional, The shared lock to use to access shared resources.
     """
 
     # pylint: disable=too-many-public-methods, too-many-instance-attributes
@@ -160,6 +164,7 @@ def __init__(
         db_session: Optional[Session] = None,
         use_whois_db: Optional[bool] = None,
         use_platform: Optional[bool] = None,
+        shared_lock: Optional[Any] = None,
     ) -> None:
         self.dns_query_tool = DNSQueryTool()
         self.whois_query_tool = WhoisQueryTool()
@@ -226,6 +231,7 @@ def __init__(
             do_syntax_check_first=do_syntax_check_first,
             db_session=db_session,
             use_platform=use_platform,
+            shared_lock=shared_lock,
         )
 
     @property
@@ -761,7 +767,9 @@ def try_to_query_status_from_whois(
         if (
             PyFunceble.facility.ConfigLoader.is_already_loaded() and self.use_whois_db
         ):  # pragma: no cover ## Not interesting enough to spend time on it.
-            whois_object = get_whois_dataset_object(db_session=self.db_session)
+            whois_object = get_whois_dataset_object(
+                db_session=self.db_session, shared_lock=self.shared_lock
+            )
             known_record = whois_object[self.subject]
 
             if known_record and not isinstance(known_record, dict):

PyFunceble/checker/availability/domain.py

@@ -115,7 +115,9 @@ def try_to_query_status_from_reputation(self) -> "DomainAvailabilityChecker":
             self.status.idna_subject,
         )
 
-        lookup_result = DomainReputationChecker(self.status.idna_subject).get_status()
+        lookup_result = DomainReputationChecker(
+            self.status.idna_subject, shared_lock=self.shared_lock
+        ).get_status()
 
         # pylint: disable=no-member
         if lookup_result and lookup_result.is_malicious():

PyFunceble/checker/availability/domain_and_ip.py

@@ -112,6 +112,7 @@ def query_status(
                 db_session=self.db_session,
                 use_whois_db=self.use_whois_db,
                 use_platform=self.use_platform,
+                shared_lock=self.shared_lock,
             )
         else:
             query_object = DomainAvailabilityChecker(
@@ -126,6 +127,7 @@ def query_status(
                 db_session=self.db_session,
                 use_whois_db=self.use_whois_db,
                 use_platform=self.use_platform,
+                shared_lock=self.shared_lock,
             )
 
         query_object.dns_query_tool = self.dns_query_tool

PyFunceble/checker/base.py

@@ -52,7 +52,7 @@
 
 import datetime
 import functools
-from typing import Optional
+from typing import Any, Optional
 
 import domain2idna
 from sqlalchemy.orm import Session
@@ -77,6 +77,12 @@ class CheckerBase:
 
         .. warning::
             This does not apply to the syntax checker - itself.
+    :param Session db_session:
+        Optional, The database session to use.
+    :param bool use_platform:
+        Optional, Authorizes the usage of the platform.
+    :param Any shared_lock:
+        Optional, The shared lock to use to access shared resources.
     """
 
     STD_DO_SYNTAX_CHECK_FIRST: bool = False
@@ -97,13 +103,16 @@ class CheckerBase:
     status: Optional[CheckerStatusBase] = None
     params: Optional[CheckerParamsBase] = None
 
+    shared_lock: Optional[Any] = None
+
     def __init__(
         self,
         subject: Optional[str] = None,
         *,
         do_syntax_check_first: Optional[bool] = None,
         db_session: Optional[Session] = None,
         use_platform: Optional[bool] = None,
+        shared_lock: Optional[Any] = None,
     ) -> None:
         self.platform_query_tool = PlatformQueryTool()
         self.url2netloc = Url2Netloc()
@@ -115,6 +124,9 @@ def __init__(
         if self.status is None:
             self.status = CheckerStatusBase()
 
+        if shared_lock is not None:
+            self.shared_lock = shared_lock
+
         if subject is not None:
             self.subject = subject
 

PyFunceble/checker/reputation/base.py

@@ -50,7 +50,7 @@
     limitations under the License.
 """
 
-from typing import List, Optional
+from typing import Any, List, Optional
 
 from sqlalchemy.orm import Session
 
@@ -75,6 +75,12 @@ class ReputationCheckerBase(CheckerBase):
     :param bool do_syntax_check_first:
         Optional, Activates/Disables the check of the status before the actual
         status gathering.
+    :param Session db_session:
+        Optional, The database session to use.
+    :param bool use_platform:
+        Optional, Authorizes the usage of the platform.
+    :param Any shared_lock:
+        Optional, The shared lock to use to access shared resources.
     """
 
     dns_query_tool: Optional[DNSQueryTool] = None
@@ -92,9 +98,10 @@ def __init__(
         do_syntax_check_first: Optional[bool] = None,
         db_session: Optional[Session] = None,
         use_platform: Optional[bool] = None,
+        shared_lock: Optional[Any] = None,
     ) -> None:
         self.dns_query_tool = DNSQueryTool()
-        self.ipv4_reputation_query_tool = IPV4ReputationDataset()
+        self.ipv4_reputation_query_tool = IPV4ReputationDataset(shared_lock=shared_lock)
         self.domain_syntax_checker = DomainSyntaxChecker()
         self.ip_syntax_checker = IPSyntaxChecker()
         self.url_syntax_checker = URLSyntaxChecker()
@@ -110,6 +117,7 @@ def __init__(
             do_syntax_check_first=do_syntax_check_first,
             db_session=db_session,
             use_platform=use_platform,
+            shared_lock=shared_lock,
         )
 
     @staticmethod

PyFunceble/checker/syntax/base.py

@@ -50,7 +50,7 @@
     limitations under the License.
 """
 
-from typing import Optional
+from typing import Any, Optional
 
 from sqlalchemy.orm import Session
 
@@ -78,13 +78,16 @@ def __init__(
         self,
         subject: Optional[str] = None,
         db_session: Optional[Session] = None,
+        shared_lock: Optional[Any] = None,
     ) -> None:
         self.params = SyntaxCheckerParams()
 
         self.status = SyntaxCheckerStatus()
         self.status.params = self.params
 
-        super().__init__(subject=subject, db_session=db_session)
+        super().__init__(
+            subject=subject, db_session=db_session, shared_lock=shared_lock
+        )
 
     def subject_propagator(self) -> "CheckerBase":
         """

PyFunceble/checker/utils/whois.py

@@ -50,7 +50,7 @@
     limitations under the License.
 """
 
-from typing import Optional, Union
+from typing import Any, Optional, Union
 
 from sqlalchemy.orm import Session
 
@@ -64,13 +64,17 @@
 
 
 def get_whois_dataset_object(
-    *, db_session: Optional[Session] = None
+    *,
+    db_session: Optional[Session] = None,
+    shared_lock: Optional[Any] = None,
 ) -> Union[DatasetBase, CSVDatasetBase, DBDatasetBase]:
     """
     Provides the whois dataset object to work with.
 
     :param db_session:
         A database session to use.
+    :param shared_lock:
+        Optional, The shared lock to use to access shared resources.
 
     :raise ValueError:
         When the given database type is unknown.
@@ -80,7 +84,7 @@ def get_whois_dataset_object(
 
     if PyFunceble.facility.ConfigLoader.is_already_loaded():
         if PyFunceble.storage.CONFIGURATION.cli_testing.db_type == "csv":
-            return CSVWhoisDataset().set_authorized(
+            return CSVWhoisDataset(shared_lock=shared_lock).set_authorized(
                 bool(PyFunceble.storage.CONFIGURATION.cli_testing.whois_db)
             )
 

PyFunceble/cli/processes/base.py

@@ -60,3 +60,10 @@ class ProcessesManagerBase(PyFunceble.ext.process_manager.ProcessManagerCore):
     """
     Provides the base of all classes.
     """
+
+    def __init__(self, *args, **kwargs) -> None:
+        # Better handling of shared locks.
+        # This help be be retro-compatible. Allowing existing users to
+        # not get exceptions.
+        shared_lock = kwargs.pop("shared_lock", None)
+        super().__init__(*args, **kwargs, shared_lock=shared_lock)

PyFunceble/cli/processes/workers/producer.py

@@ -125,9 +125,15 @@ def perform_external_poweron_checks(self) -> None:
         self.file_printer = FilePrinter(
             skip_column=skip_columns, extra_formatters=extra_formatters
         )
-        self.whois_dataset = get_whois_dataset_object(db_session=self.db_session)
-        self.inactive_dataset = get_inactive_dataset_object(db_session=self.db_session)
-        self.continue_dataset = get_continue_dataset_object(db_session=self.db_session)
+        self.whois_dataset = get_whois_dataset_object(
+            db_session=self.db_session, shared_lock=self.shared_lock
+        )
+        self.inactive_dataset = get_inactive_dataset_object(
+            db_session=self.db_session, shared_lock=self.shared_lock
+        )
+        self.continue_dataset = get_continue_dataset_object(
+            db_session=self.db_session, shared_lock=self.shared_lock
+        )
         self.status_file_generator = StatusFileGenerator().guess_all_settings()
         self.counter = FilesystemCounter()
         self.registrar_counter = RegistrarCounter()

PyFunceble/cli/processes/workers/tester.py

@@ -90,12 +90,12 @@ def perform_external_poweron_checks(self) -> None:
 
         self.continue_dataset = (
             PyFunceble.cli.utils.testing.get_continue_dataset_object(
-                db_session=self.db_session
+                db_session=self.db_session, shared_lock=self.shared_lock
             )
         )
         self.inactive_dataset = (
             PyFunceble.cli.utils.testing.get_inactive_dataset_object(
-                db_session=self.db_session
+                db_session=self.db_session, shared_lock=self.shared_lock
             )
         )
 
@@ -191,7 +191,7 @@ def _init_testing_object(
             self.initiated_testing_objects[checker_type][
                 subject_type
             ] = self.known_testing_objects[checker_type][subject_type](
-                db_session=self.db_session
+                db_session=self.db_session, shared_lock=self.shared_lock
             ).set_do_syntax_check_first(
                 # We want to always check the syntax first (ONLY UNDER THE CLI)
                 not bool(PyFunceble.storage.CONFIGURATION.cli_testing.local_network)
@@ -278,12 +278,12 @@ def target(self, consumed: dict) -> Optional[Tuple[Any, ...]]:
             test_dataset["idna_subject"],
         )
 
-        self._init_testing_object(
+        testing_object = self._init_testing_object(
             test_dataset["subject_type"], test_dataset["checker_type"]
         )
 
         result = (
-            self.testing_object.set_subject(test_dataset["idna_subject"])
+            testing_object.set_subject(test_dataset["idna_subject"])
             .query_status()
             .get_status()
         )