Merge pull request #593 from MohannadAK/fix-use-iterator

Fixes Issue #587

Author: JonoYang

packagedb/models.py

@@ -71,19 +71,6 @@ def get_or_none(self, *args, **kwargs):
         except Package.DoesNotExist:
             return
 
-    def paginated(self, per_page=5000):
-        """
-        Iterate over a (large) QuerySet by chunks of ``per_page`` items.
-        This technique is essential for preventing memory issues when iterating
-        See these links for inspiration:
-        https://nextlinklabs.com/resources/insights/django-big-data-iteration
-        https://stackoverflow.com/questions/4222176/why-is-iterating-through-a-large-django-queryset-consuming-massive-amounts-of-me/
-        """
-        paginator = Paginator(self, per_page=per_page)
-        for page_number in paginator.page_range:
-            page = paginator.page(page_number)
-            yield from page.object_list
-
 
 VCS_CHOICES = [
     ("git", "git"),

purl2vcs/src/purl2vcs/find_source_repo.py

@@ -188,7 +188,7 @@ def get_source_package_for_all_packages():
     Add the PackageURL of the source repository of a Package
     if found
     """
-    for package in Package.objects.all().paginated():
+    for package in Package.objects.all().iterator():
         get_source_package_and_add_to_package_set(package)