handle subprocesses closing db connections

Author: furlongm

errata/sources/distros/alma.py

@@ -17,6 +17,8 @@
 import concurrent.futures
 import json
 
+from django.db import connections
+
 from operatingsystems.utils import get_or_create_osrelease
 from packages.models import Package
 from packages.utils import get_or_create_package, parse_package_string
@@ -74,6 +76,7 @@ def process_alma_errata_serially(release, advisories):
 def process_alma_errata_concurrently(release, advisories):
     """ Process Alma Linux Errata concurrently
     """
+    connections.close_all()
     elen = len(advisories)
     pbar_start.send(sender=None, ptext=f'Processing {elen} Alma {release} Errata', plen=elen)
     i = 0

errata/sources/distros/arch.py

@@ -17,6 +17,8 @@
 import concurrent.futures
 import json
 
+from django.db import connections
+
 from operatingsystems.utils import get_or_create_osrelease
 from patchman.signals import error_message, pbar_start, pbar_update
 from packages.models import Package
@@ -66,6 +68,7 @@ def parse_arch_errata_concurrently(advisories):
     """ Parse Arch Linux Errata Advisories concurrently
     """
     osrelease = get_or_create_osrelease(name='Arch Linux')
+    connections.close_all()
     elen = len(advisories)
     pbar_start.send(sender=None, ptext=f'Processing {elen} Arch Advisories', plen=elen)
     i = 0

errata/sources/distros/debian.py

@@ -21,6 +21,8 @@
 from debian.deb822 import Dsc
 from io import StringIO
 
+from django.db import connections
+
 from operatingsystems.models import OSRelease
 from operatingsystems.utils import get_or_create_osrelease
 from packages.models import Package
@@ -181,6 +183,7 @@ def create_debian_errata_serially(errata, accepted_codenames):
 def create_debian_errata_concurrently(errata, accepted_codenames):
     """ Create Debian Errata concurrently
     """
+    connections.close_all()
     elen = len(errata)
     pbar_start.send(sender=None, ptext=f'Processing {elen} Debian Errata', plen=elen)
     i = 0

errata/sources/distros/rocky.py

@@ -18,6 +18,7 @@
 import concurrent.futures
 from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential
 
+from django.db import connections
 from django.db.utils import OperationalError
 
 from operatingsystems.utils import get_or_create_osrelease
@@ -158,6 +159,7 @@ def process_rocky_errata_serially(advisories):
 def process_rocky_errata_concurrently(advisories):
     """ Process Rocky Linux errata concurrently
     """
+    connections.close_all()
     elen = len(advisories)
     pbar_start.send(sender=None, ptext=f'Processing {elen} Rocky Errata', plen=elen)
     i = 0

errata/sources/distros/ubuntu.py

@@ -21,6 +21,8 @@
 from io import StringIO
 from urllib.parse import urlparse
 
+from django.db import connections
+
 from operatingsystems.models import OSRelease, OSVariant
 from operatingsystems.utils import get_or_create_osrelease
 from packages.models import Package
@@ -87,6 +89,7 @@ def parse_usn_data_serially(advisories, accepted_releases):
 def parse_usn_data_concurrently(advisories, accepted_releases):
     """ Parse the Ubuntu USN data concurrently
     """
+    connections.close_all()
     elen = len(advisories)
     pbar_start.send(sender=None, ptext=f'Processing {elen} Ubuntu Errata', plen=elen)
     i = 0

errata/sources/repos/yum.py

@@ -18,6 +18,8 @@
 from io import BytesIO
 from defusedxml import ElementTree
 
+from django.db import connections
+
 from operatingsystems.utils import get_or_create_osrelease
 from packages.models import Package
 from packages.utils import get_or_create_package
@@ -55,6 +57,7 @@ def extract_updateinfo_serially(updates, elen):
 def extract_updateinfo_concurrently(updates, elen):
     """ Parses updateinfo.xml and extracts package/errata information concurrently
     """
+    connections.close_all()
     pbar_start.send(sender=None, ptext=f'Extracting {elen} updateinfo Errata', plen=elen)
     i = 0
     with concurrent.futures.ProcessPoolExecutor(max_workers=100) as executor:

etc/patchman/local_settings.py

@@ -35,6 +35,9 @@
 # Maximum number of mirrors to add or refresh per repo
 MAX_MIRRORS = 2
 
+# Maximum number of failures before disabling a mirror, set to -1 to never disable mirrors
+MAX_MIRROR_FAILURES = 14
+
 # Number of days to wait before raising that a host has not reported
 DAYS_WITHOUT_REPORT = 14