Improve performance of API calls #1538 (#1547)

* Add pyinstrument to debug mode

Reference:#1538
Signed-off-by: Philippe Ombredanne <[email protected]>

* Apply minor cosmetic models.py changes

Remove unused imports
Move univers monkey patching to top level

Reference:#1538
Signed-off-by: Philippe Ombredanne <[email protected]>

* Annotate queryset for is_vulnerable

* is_vulnerable is now a queryset annotation.
* with_is_vulnerable Package manager method annotates the queryset
* It is now called anywhere needed.

The results with a single API call to
http://127.0.0.1:8001/api/packages/63 are:
* before 1.7 sec
* after  0.25 sec

Or an improvement where the old is 6.8 times slower than the new.

Reference:#1538
Signed-off-by: Philippe Ombredanne <[email protected]>

* Improve sorting of packages by version

* simplify sorting code, revert to using lists
* use cached_property where relevant

Reference:#1538
Signed-off-by: Philippe Ombredanne <[email protected]>

* Annotate queryset for is_vulnerable everywhere

* Fix how we find vulnerable and non vulnerable versions
* Complete using is_vulnerable in relevant querysets.
* Adjust tests for clarity and DRY using small method to make the tests
  setup easier to read
* Use proper typing annotation for PackageURL or str
* Add new Package manager from_purl() method to create a Package from
  a PURL

Reference:#1538
Signed-off-by: Philippe Ombredanne <[email protected]>

* Do not print things in tests

Signed-off-by: Philippe Ombredanne <[email protected]>

---------

Signed-off-by: Philippe Ombredanne <[email protected]>

Author: pombredanne

setup.cfg

@@ -118,6 +118,7 @@ dev =
     commoncode
     # debug
     django-debug-toolbar
+    pyinstrument
 
 [options.entry_points]
 console_scripts =

vulnerabilities/api.py

@@ -108,6 +108,8 @@ def get_vulnerability(self, vuln):
 
     purl = serializers.CharField(source="package_url")
 
+    is_vulnerable = serializers.BooleanField()
+
     class Meta:
         model = Package
         fields = ["url", "purl", "is_vulnerable", "affected_by_vulnerabilities"]
@@ -248,21 +250,27 @@ def get_latest_non_vulnerable(self, package):
 
     affected_by_vulnerabilities = serializers.SerializerMethodField("get_affected_vulnerabilities")
 
-    fixing_vulnerabilities = serializers.SerializerMethodField("get_fixed_vulnerabilities")
+    fixing_vulnerabilities = serializers.SerializerMethodField("get_fixing_vulnerabilities")
+
+    is_vulnerable = serializers.BooleanField()
 
     def get_fixed_packages(self, package):
         """
         Return a queryset of all packages that fix a vulnerability with
         same type, namespace, name, subpath and qualifiers of the `package`
         """
-        return Package.objects.filter(
-            name=package.name,
-            namespace=package.namespace,
-            type=package.type,
-            qualifiers=package.qualifiers,
-            subpath=package.subpath,
-            packagerelatedvulnerability__fix=True,
-        ).distinct()
+        return (
+            Package.objects.filter(
+                name=package.name,
+                namespace=package.namespace,
+                type=package.type,
+                qualifiers=package.qualifiers,
+                subpath=package.subpath,
+                packagerelatedvulnerability__fix=True,
+            )
+            .with_is_vulnerable()
+            .distinct()
+        )
 
     def get_vulnerabilities_for_a_package(self, package, fix) -> dict:
         """
@@ -285,7 +293,7 @@ def get_vulnerabilities_for_a_package(self, package, fix) -> dict:
             context={"request": self.context["request"]},
         ).data
 
-    def get_fixed_vulnerabilities(self, package) -> dict:
+    def get_fixing_vulnerabilities(self, package) -> dict:
         """
         Return a mapping of vulnerabilities fixed in the given `package`.
         """
@@ -322,12 +330,15 @@ class Meta:
             "version",
             "qualifiers",
             "subpath",
+            "is_vulnerable",
             "next_non_vulnerable_version",
             "latest_non_vulnerable_version",
             "affected_by_vulnerabilities",
             "fixing_vulnerabilities",
         ]
 
+    is_vulnerable = serializers.BooleanField()
+
 
 class PackageFilterSet(filters.FilterSet):
     purl = filters.CharFilter(method="filter_purl")
@@ -390,6 +401,9 @@ class PackageViewSet(viewsets.ReadOnlyModelViewSet):
     filterset_class = PackageFilterSet
     throttle_classes = [StaffUserRateThrottle, AnonRateThrottle]
 
+    def get_queryset(self):
+        return super().get_queryset().with_is_vulnerable()
+
     @extend_schema(
         request=PackageBulkSearchRequestSerializer,
         responses={200: PackageSerializer(many=True)},
@@ -436,6 +450,7 @@ def bulk_search(self, request):
                 Package.objects.filter(plain_package_url__in=plain_purls)
                 .order_by("plain_package_url")
                 .distinct("plain_package_url")
+                .with_is_vulnerable()
             )
 
             if not purl_only:
@@ -449,7 +464,7 @@ def bulk_search(self, request):
             vulnerable_purls = [str(package.plain_package_url) for package in vulnerable_purls]
             return Response(data=vulnerable_purls)
 
-        query = Package.objects.filter(package_url__in=purls).distinct()
+        query = Package.objects.filter(package_url__in=purls).distinct().with_is_vulnerable()
 
         if not purl_only:
             return Response(PackageSerializer(query, many=True, context={"request": request}).data)
@@ -463,7 +478,9 @@ def all(self, request):
         """
         Return the Package URLs of all packages known to be vulnerable.
         """
-        vulnerable_packages = Package.objects.vulnerable().only("package_url").distinct()
+        vulnerable_packages = (
+            Package.objects.vulnerable().only("package_url").distinct().with_is_vulnerable()
+        )
         vulnerable_purls = [str(package.package_url) for package in vulnerable_packages]
         return Response(vulnerable_purls)
 
@@ -494,11 +511,8 @@ def lookup(self, request):
         validated_data = serializer.validated_data
         purl = validated_data.get("purl")
 
-        return Response(
-            PackageSerializer(
-                Package.objects.for_purls([purl]), many=True, context={"request": request}
-            ).data
-        )
+        qs = self.get_queryset().for_purls([purl]).with_is_vulnerable()
+        return Response(PackageSerializer(qs, many=True, context={"request": request}).data)
 
     @extend_schema(
         request=PackageurlListSerializer,
@@ -529,7 +543,7 @@ def bulk_lookup(self, request):
 
         return Response(
             PackageSerializer(
-                Package.objects.for_purls(purls),
+                Package.objects.for_purls(purls).with_is_vulnerable(),
                 many=True,
                 context={"request": request},
             ).data
@@ -547,33 +561,47 @@ class VulnerabilityViewSet(viewsets.ReadOnlyModelViewSet):
     Lookup for vulnerabilities affecting packages.
     """
 
+    queryset = Vulnerability.objects.all()
+
     def get_fixed_packages_qs(self):
         """
         Filter the packages that fixes a vulnerability
         on fields like name, namespace and type.
         """
-        package_filter_data = {"packagerelatedvulnerability__fix": True}
+        return self.get_packages_qs().filter(packagerelatedvulnerability__fix=True)
 
+    def get_packages_qs(self):
+        """
+        Filter the packages on type, namespace and name.
+        """
         query_params = self.request.query_params
-        for field_name in ["name", "namespace", "type"]:
-            value = query_params.get(field_name)
-            if value:
+        package_filter_data = {}
+        for field_name in ("type", "namespace", "name"):
+            if value := query_params.get(field_name):
                 package_filter_data[field_name] = value
 
-        return PackageFilterSet(package_filter_data).qs
+        return PackageFilterSet(package_filter_data).qs.with_is_vulnerable()
 
     def get_queryset(self):
         """
         Assign filtered packages queryset from `get_fixed_packages_qs`
         to a custom attribute `filtered_fixed_packages`
         """
-        return Vulnerability.objects.prefetch_related(
-            "weaknesses",
-            Prefetch(
-                "packages",
-                queryset=self.get_fixed_packages_qs(),
-                to_attr="filtered_fixed_packages",
-            ),
+        return (
+            super()
+            .get_queryset()
+            .prefetch_related(
+                Prefetch(
+                    "packages",
+                    queryset=self.get_packages_qs(),
+                ),
+                "weaknesses",
+                Prefetch(
+                    "packages",
+                    queryset=self.get_fixed_packages_qs(),
+                    to_attr="filtered_fixed_packages",
+                ),
+            )
         )
 
     serializer_class = VulnerabilitySerializer