Merge pull request #678 from furlongm/errata-speedups

speed up errata processing

Author: furlongm

errata/utils.py

@@ -14,6 +14,10 @@
 # You should have received a copy of the GNU General Public License
 # along with Patchman. If not, see <http://www.gnu.org/licenses/>
 
+import concurrent.futures
+
+from django.db import connections
+
 from util import tz_aware_datetime
 from errata.models import Erratum
 from packages.models import PackageUpdate
@@ -61,24 +65,37 @@ def mark_errata_security_updates():
     """ For each set of erratum packages, modify any PackageUpdate that
         should be marked as a security update.
     """
+    connections.close_all()
     elen = Erratum.objects.count()
-    pbar_start.send(sender=None, ptext=f'Scanning {elen} Errata', plen=elen)
-    for i, e in enumerate(Erratum.objects.all()):
-        pbar_update.send(sender=None, index=i + 1)
-        e.scan_for_security_updates()
+    pbar_start.send(sender=None, ptext=f'Scanning {elen} Errata for security updates', plen=elen)
+    i = 0
+    with concurrent.futures.ProcessPoolExecutor(max_workers=25) as executor:
+        futures = [executor.submit(e.scan_for_security_updates) for e in Erratum.objects.all()]
+        for future in concurrent.futures.as_completed(futures):
+            pbar_update.send(sender=None, index=i + 1)
+            i += 1
 
 
 def scan_package_updates_for_affected_packages():
     """ Scan PackageUpdates for packages affected by errata
     """
-    for pu in PackageUpdate.objects.all():
+    plen = PackageUpdate.objects.count()
+    pbar_start.send(sender=None, ptext=f'Scanning {plen} Updates for affected packages', plen=plen)
+    for i, pu in enumerate(PackageUpdate.objects.all()):
+        pbar_update.send(sender=None, index=i + 1)
         for e in pu.newpackage.provides_fix_in_erratum.all():
             e.affected_packages.add(pu.oldpackage)
 
 
-def add_errata_affected_packages():
+def enrich_errata():
+    """ Enrich Errata with data from osv.dev
+    """
+    connections.close_all()
     elen = Erratum.objects.count()
-    pbar_start.send(sender=None, ptext=f'Adding affected packages to {elen} Errata', plen=elen)
-    for i, e in enumerate(Erratum.objects.all()):
-        pbar_update.send(sender=None, index=i + 1)
-        e.fetch_osv_dev_data()
+    pbar_start.send(sender=None, ptext=f'Adding osv.dev data to {elen} Errata', plen=elen)
+    i = 0
+    with concurrent.futures.ProcessPoolExecutor(max_workers=25) as executor:
+        futures = [executor.submit(e.fetch_osv_dev_data) for e in Erratum.objects.all()]
+        for future in concurrent.futures.as_completed(futures):
+            pbar_update.send(sender=None, index=i + 1)
+            i += 1

sbin/patchman

@@ -30,7 +30,8 @@ from django.conf import settings  # noqa
 django_setup()
 
 from arch.utils import clean_architectures
-from errata.utils import mark_errata_security_updates, add_errata_affected_packages
+from errata.utils import mark_errata_security_updates, enrich_errata, \
+    scan_package_updates_for_affected_packages
 from errata.tasks import update_errata
 from hosts.models import Host
 from modules.utils import clean_modules
@@ -522,8 +523,9 @@ def process_args(args):
         showhelp = False
     if args.update_errata:
         update_errata(args.erratum_type, args.force, args.repo)
+        scan_package_updates_for_affected_packages()
         mark_errata_security_updates()
-        add_errata_affected_packages()
+        enrich_errata()
         showhelp = False
     if args.update_cves:
         update_cves(args.cve, args.fetch_nist_data)