test redhat importer performance by profiling

Signed-off-by: ziad <[email protected]>

Signed-off-by: ziad <[email protected]>

Author: ziadhany

vulnerabilities/management/commands/import.py

@@ -6,7 +6,6 @@
 # See https://github.com/nexB/vulnerablecode for support or download.
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
-import threading
 import traceback
 
 from django.core.management.base import BaseCommand
@@ -42,10 +41,6 @@ def handle(self, *args, **options):
             self.import_data(IMPORTERS_REGISTRY.values())
             return
 
-        if options["all_in_parallel"]:
-            self.import_data_in_parallel(IMPORTERS_REGISTRY.values())
-            return
-
         sources = options["sources"]
         if not sources:
             raise CommandError('Please provide at least one importer to run or use "--all".')
@@ -85,36 +80,6 @@ def import_data(self, importers):
         if failed_importers:
             raise CommandError(f"{len(failed_importers)} failed!: {','.join(failed_importers)}")
 
-    def import_data_in_parallel(self, importers):
-        failed_importers = []
-        thread_list = []
-        for importer in importers:
-            self.stdout.write(f"Importing data using {importer.qualified_name}")
-            try:
-                thread = threading.Thread(
-                    target=ImportRunner(importer).run(), name=importer.qualified_name
-                )
-                thread.start()
-                thread_list.append(thread)
-            except Exception:
-                failed_importers.append(importer.qualified_name)
-                traceback.print_exc()
-                self.stdout.write(
-                    self.style.ERROR(
-                        f"Failed to run importer {importer.qualified_name}. Continuing..."
-                    )
-                )
-        for thread in thread_list:
-            thread.join()
-
-        success_list = {value for value in IMPORTERS_REGISTRY if value not in failed_importers}
-        if success_list:
-            self.stdout.write(
-                self.style.SUCCESS(f"Successfully imported data using {success_list} ")
-            )
-        if failed_importers:
-            raise CommandError(f"{len(failed_importers)} failed!: {','.join(failed_importers)}")
-
 
 def validate_importers(sources):
     importers = []

vulnerabilities/tests/test_performance.py

@@ -0,0 +1,33 @@
+#
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+import pytest
+
+# this import are used in the script
+from vulnerabilities.importers import redhat
+
+script = """for i, data in enumerate(redhat.RedhatImporter().advisory_data()):
+    if 1 == 100: 
+        break"""
+
+
+@pytest.mark.skip("Use only for local profiling")
+@pytest.mark.django_db
+class TestImporter:
+    def test_redhat_importer_performance_profiling(self):
+        print_profiling_status(script, "redhat.txt")
+
+
+def print_profiling_status(test_py, stats_file, top=50):
+    import cProfile as profile
+    import pstats
+
+    profile.runctx(test_py, globals(), locals(), stats_file)
+    p = pstats.Stats(stats_file)
+    p.sort_stats("time").print_stats(top)