add on demand package data collection for golang, gitlab and bitbucket #596 (#608)

Author: chinyeungli

minecode/collectors/bitbucket.py

@@ -0,0 +1,50 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# purldb is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import logging
+
+import requests
+
+
+"""
+Collect bitbucket packages from bitbucket registries.
+"""
+
+logger = logging.getLogger(__name__)
+handler = logging.StreamHandler()
+logger.addHandler(handler)
+logger.setLevel(logging.INFO)
+
+
+def bitbucket_get_all_package_version_author(subset_path):
+    """
+    Return a list of all version numbers along with author for the package.
+    """
+    repo_tags = f"https://api.bitbucket.org/2.0/repositories/{subset_path}/refs/tags"
+    version_author_list = []
+    try:
+        while repo_tags:
+            response = requests.get(repo_tags)
+            response.raise_for_status()
+            data = response.json()
+            if data["size"] > 0:
+                # Get all available versions
+                for item in data["values"]:
+                    version = item.get("name")
+                    target = item.get("target") or {}
+                    author = target.get("author") or {}
+                    if author.get("type") == "author":
+                        user = author.get("user") or {}
+                        author_display_name = user.get("display_name")
+                    version_author_list.append((version, author_display_name))
+            # Handle pagination
+            repo_tags = data.get("next", None)
+        return version_author_list
+    except requests.exceptions.HTTPError as err:
+        logger.error(f"HTTP error occurred: {err}")

minecode/collectors/generic.py

@@ -100,7 +100,7 @@ def packagedata_from_dict(package_data):
     return PackageData.from_data(cleaned_package_data)
 
 
-def map_fetchcode_supported_package(package_url, pipelines, priority=0):
+def map_fetchcode_supported_package(package_url, pipelines, priority=0, from_go_lang=False):
     """
     Add a `package_url` supported by fetchcode to the PackageDB.
 
@@ -109,13 +109,24 @@ def map_fetchcode_supported_package(package_url, pipelines, priority=0):
     from minecode.model_utils import add_package_to_scan_queue
     from minecode.model_utils import merge_or_create_package
 
-    packages = [p for p in info(str(package_url)) or []]
+    try:
+        packages = []
+        packages = [p for p in info(str(package_url)) or []]
+    except Exception as e:
+        print(str(e))
 
     if not packages:
-        error = f"Could not find package using fetchcode: {package_url}"
+        if from_go_lang:
+            purl = "pkg:golang/" + str(package_url).partition("pkg:")[2]
+        else:
+            purl = str(package_url)
+        error = f"Could not find package using fetchcode: {purl}"
         logger.error(error)
         return error
 
+    if from_go_lang:
+        packages[0].type = "golang"
+        packages[0].namespace = "github.com/" + packages[0].namespace
     package_data = packages[0].to_dict()
 
     # Remove obsolete Package fields see https://github.com/aboutcode-org/fetchcode/issues/108

minecode/collectors/github.py

@@ -13,6 +13,42 @@
 from minecode.collectors.generic import map_fetchcode_supported_package
 
 
+def github_get_all_versions(subset_path):
+    """
+    Fetch all versions (tags) from a GitHub repository using the API
+    Returns a list of all version tags in the repository
+    """
+    import requests
+
+    url = f"https://api.github.com/repos/{subset_path}/tags"
+    version_list = []
+    page = 1
+
+    while True:
+        response = requests.get(
+            url,
+            params={"page": page, "per_page": 100},  # Max 100 per page
+            headers={"Accept": "application/vnd.github.v3+json"},
+        )
+        response.raise_for_status()
+
+        data = response.json()
+        if not data:
+            break
+
+        for tag in data:
+            version = tag.get("name") or {}
+            if version:
+                version_list.append(version)
+        page += 1
+
+        # Check if we've reached the last page
+        if "next" not in response.links:
+            break
+
+    return version_list
+
+
 # Indexing GitHub PURLs requires a GitHub API token.
 # Please add your GitHub API key to the `.env` file, for example: `GH_TOKEN=your-github-api`.
 @priority_router.route("pkg:github/.*")

minecode/collectors/gitlab.py

@@ -0,0 +1,45 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# purldb is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import logging
+
+import requests
+
+
+"""
+Collect gitlab packages from gitlab registries.
+"""
+
+logger = logging.getLogger(__name__)
+handler = logging.StreamHandler()
+logger.addHandler(handler)
+logger.setLevel(logging.INFO)
+
+
+def gitlab_get_all_package_version_author(subset_path):
+    """
+    Return a list of all version numbers along with author and author email
+    for the package.
+    """
+    repo_tags = f"https://gitlab.com/api/v4/projects/{subset_path}/repository/tags"
+    try:
+        response = requests.get(repo_tags)
+        response.raise_for_status()
+        data = response.json()
+        version_author_list = []
+        # Get all available versions
+        for item in data:
+            version = item.get("name")
+            commit = item.get("commit") or {}
+            author = commit.get("author_name") or ""
+            author_email = commit.get("author_email") or ""
+            version_author_list.append((version, author, author_email))
+        return version_author_list
+    except requests.exceptions.HTTPError as err:
+        logger.error(f"HTTP error occurred: {err}")