Add support to mine pypi Package-URLs #662 (#677)

* Add support to mine pypi packageURLs

Signed-off-by: Ayan Sinha Mahapatra <[email protected]>

* Refactor and fix pypi miners

Reference: #662
Signed-off-by: Ayan Sinha Mahapatra <[email protected]>

* Fix code style issues

Signed-off-by: Ayan Sinha Mahapatra <[email protected]>

---------

Signed-off-by: Ayan Sinha Mahapatra <[email protected]>

Author: AyanSinhaMahapatra

.github/workflows/pypi-release-minecode-pipeline.yml

@@ -0,0 +1,38 @@
+name: Build minecode-pipeline Python distributions and publish on PyPI
+
+on:
+  workflow_dispatch:
+  push:
+   tags:
+     - "minecode-pipeline/*"
+
+jobs:
+  build-and-publish:
+    name: Build and publish library to PyPI
+    runs-on: ubuntu-22.04
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+
+      - name: Install flot
+        run: python -m pip install flot --user
+
+      - name: Build binary wheel and source tarball
+        run:  python -m flot --pyproject pyproject-minecode_pipeline.toml --sdist --wheel --output-dir dist/
+
+      - name: Publish to PyPI
+        if: startsWith(github.ref, 'refs/tags')
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.PYPI_API_TOKEN_MINECODE_PIPELINE }}
+
+      - name: Upload built archives
+        uses: actions/upload-artifact@v4
+        with:
+          name: pypi_archives
+          path: dist/*

Makefile

@@ -139,6 +139,9 @@ test_matchcode:
 
 test: test_purldb test_matchcode test_toolkit test_clearcode
 
+test_minecode:
+	${ACTIVATE} ${PYTHON_EXE} -m pytest -vvs minecode_pipeline
+
 shell:
 	${MANAGE} shell
 

minecode_pipeline/README.rst

@@ -0,0 +1,43 @@
+minecode-pipeline
+===================
+
+minecode-pipeline is an add-on library working with scancode.io to define pipelines to mine
+packageURLs and package metadata from ecosystem repositories and APIs.
+
+Installation
+------------
+
+Requirements
+############
+
+* install minecode-pipeline dependencies
+* `pip install minecode-pipeline`
+
+
+Funding
+-------
+
+This project was funded through the NGI Assure Fund https://nlnet.nl/assure, a
+fund established by NLnet https://nlnet.nl/ with financial support from the
+European Commission's Next Generation Internet programme, under the aegis of DG
+Communications Networks, Content and Technology under grant agreement No 957073.
+
+This project is also funded through grants from the Google Summer of Code
+program, continuing support and sponsoring from nexB Inc. and generous
+donations from multiple sponsors.
+
+
+License
+-------
+
+Copyright (c) nexB Inc. and others. All rights reserved.
+
+purldb is a trademark of nexB Inc.
+
+SPDX-License-Identifier: Apache-2.0
+
+minecode-pipeline is licensed under the Apache License version 2.0.
+
+See https://www.apache.org/licenses/LICENSE-2.0 for the license text.
+See https://github.com/aboutcode-org/purldb for support or download.
+See https://aboutcode.org for more information about nexB OSS projects.

minecode_pipeline/__init__.py

@@ -0,0 +1,8 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# purldb is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#

minecode_pipeline/miners/__init__.py

@@ -0,0 +1,8 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# purldb is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#

minecode_pipeline/miners/pypi.py

@@ -0,0 +1,86 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# purldb is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+
+import json
+import requests
+
+from packageurl import PackageURL
+
+from minecode_pipeline.utils import get_temp_file
+
+"""
+Visitors for Pypi and Pypi-like Python package repositories.
+
+We have this hierarchy in Pypi simple/ index:
+    pypi projects (JSON/HTML) -> project versions (JSON/HTML) -> download urls
+
+https://pypi.org/simple/
+Pypi serves a main index via JSON/HTML API that contains a list of package names
+and some info on when a package was updated by releasing a new version.
+See https://docs.pypi.org/api/index-api/ for more details.
+This index also has a list of versions and download URLs of all
+uploaded/available package archives and some basic metadata.
+
+https://pypi.org/pypi/{name}/json
+For each package, a JSON contains details including the list of all releases
+and archives, their URLs, and some metadata for each release.
+For each release, a JSON contains details for the released version and all the
+downloads available for this release.
+"""
+
+
+pypi_json_headers = {"Accept": "application/vnd.pypi.simple.v1+json"}
+
+
+PYPI_REPO = "https://pypi.org/simple/"
+PYPI_TYPE = "pypi"
+
+
+def get_pypi_packages(pypi_repo, logger=None):
+    response = requests.get(pypi_repo, headers=pypi_json_headers)
+    if not response.ok:
+        return
+
+    packages = response.json()
+    temp_file = get_temp_file("PypiPackagesJSON")
+    with open(temp_file, "w", encoding="utf-8") as f:
+        json.dump(packages, f, indent=4)
+
+    return temp_file
+
+
+def get_pypi_packageurls(name):
+    packageurls = []
+
+    project_index_api_url = PYPI_REPO + name
+    response = requests.get(project_index_api_url, headers=pypi_json_headers)
+    if not response.ok:
+        return packageurls
+
+    project_data = response.json()
+    for version in project_data.get("versions"):
+        purl = PackageURL(
+            type=PYPI_TYPE,
+            name=name,
+            version=version,
+        )
+        packageurls.append(purl.to_string())
+
+    return packageurls
+
+
+def load_pypi_packages(packages):
+    with open(packages) as f:
+        packages_data = json.load(f)
+
+    last_serial = packages_data.get("meta").get("_last-serial")
+    packages = packages_data.get("projects")
+
+    return last_serial, packages

minecode_pipeline/pipelines/__init__.py

@@ -0,0 +1,8 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# purldb is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#

minecode_pipeline/pipelines/mine_pypi.py

@@ -0,0 +1,56 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# http://nexb.com and https://github.com/aboutcode-org/scancode.io
+# The ScanCode.io software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode.io is provided as-is without warranties.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+# ScanCode.io should be considered or used as legal advice. Consult an Attorney
+# for any legal advice.
+#
+# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
+# Visit https://github.com/aboutcode-org/scancode.io for support and download.
+
+from scanpipe.pipelines import Pipeline
+from scanpipe.pipes import federatedcode
+
+from minecode_pipeline.pipes import pypi
+
+
+class MineandPublishPypiPURLs(Pipeline):
+    """
+    Mine all packageURLs from a pypi index and publish them to
+    a FederatedCode repo.
+    """
+
+    @classmethod
+    def steps(cls):
+        return (
+            cls.check_federatedcode_eligibility,
+            cls.mine_pypi_packages,
+            cls.mine_and_publish_pypi_packageurls,
+        )
+
+    def check_federatedcode_eligibility(self):
+        """
+        Check if the project fulfills the following criteria for
+        pushing the project result to FederatedCode.
+        """
+        federatedcode.check_federatedcode_configured_and_available()
+
+    def mine_pypi_packages(self):
+        """Mine pypi package names from pypi indexes."""
+        self.pypi_packages = pypi.mine_pypi_packages(logger=self.log)
+
+    def mine_and_publish_pypi_packageurls(self):
+        """Get pypi packageURLs for all mined pypi package names."""
+        pypi.mine_and_publish_pypi_packageurls(packages=self.pypi_packages, logger=self.log)

minecode_pipeline/pipes/__init__.py

@@ -0,0 +1,8 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# purldb is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#