Create minecode_pipeline module for mine cargo

Signed-off-by: ziad hany <[email protected]>

Author: ziadhany

minecode_pipeline/README.rst

@@ -0,0 +1,43 @@
+minecode-pipeline
+===================
+
+minecode-pipeline is an add-on library working with scancode.io to define pipelines to mine
+packageURLs and package metadata from ecosystem repositories and APIs.
+
+Installation
+------------
+
+Requirements
+############
+
+* install minecode-pipeline dependencies
+* `pip install minecode-pipeline`
+
+
+Funding
+-------
+
+This project was funded through the NGI Assure Fund https://nlnet.nl/assure, a
+fund established by NLnet https://nlnet.nl/ with financial support from the
+European Commission's Next Generation Internet programme, under the aegis of DG
+Communications Networks, Content and Technology under grant agreement No 957073.
+
+This project is also funded through grants from the Google Summer of Code
+program, continuing support and sponsoring from nexB Inc. and generous
+donations from multiple sponsors.
+
+
+License
+-------
+
+Copyright (c) nexB Inc. and others. All rights reserved.
+
+purldb is a trademark of nexB Inc.
+
+SPDX-License-Identifier: Apache-2.0
+
+minecode-pipeline is licensed under the Apache License version 2.0.
+
+See https://www.apache.org/licenses/LICENSE-2.0 for the license text.
+See https://github.com/aboutcode-org/purldb for support or download.
+See https://aboutcode.org for more information about nexB OSS projects.

minecode_pipeline/__init__.py

@@ -0,0 +1,8 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# purldb is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#

minecode_pipeline/pipelines/__init__.py

@@ -0,0 +1,8 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# purldb is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#

minecode_pipeline/pipelines/mine_cargo.py

@@ -0,0 +1,71 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# http://nexb.com and https://github.com/aboutcode-org/scancode.io
+# The ScanCode.io software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode.io is provided as-is without warranties.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+# ScanCode.io should be considered or used as legal advice. Consult an Attorney
+# for any legal advice.
+#
+# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
+# Visit https://github.com/aboutcode-org/scancode.io for support and download.
+import json
+from pathlib import Path
+
+from minecode_pipeline.pipes import cargo
+from scanpipe.pipelines.publish_to_federatedcode import PublishToFederatedCode
+from fetchcode.vcs import fetch_via_vcs
+
+
+class MineCargo(PublishToFederatedCode):
+    """Pipeline to mine Cargo (crates.io) packages and publish them to FederatedCode."""
+
+    repo_url = "git+https://github.com/rust-lang/crates.io-index"
+
+    @classmethod
+    def steps(cls):
+        return (
+            cls.check_federatedcode_eligibility,
+            cls.clone_cargo_index,
+            cls.clone_repository,
+            cls.collect_packages_from_cargo,
+            cls.delete_local_clone,
+        )
+
+    def clone_cargo_index(self, repo_url):
+        """
+        Clone the repo at repo_url and return the VCSResponse object
+        """
+        self.vcs_response = fetch_via_vcs(repo_url)
+
+    def collect_packages_from_cargo(self):
+        base_path = Path(self.vcs_response.dest_dir)
+
+        json_files = []
+        for file_path in base_path.glob("**/*"):
+            if not file_path.is_file():
+                continue
+            if file_path.name in {"config.json", "README.md", "update-dl-url.yml"}:
+                continue
+            json_files.append(file_path)
+
+        for idx, file_path in enumerate(json_files, start=1):
+            try:
+                with open(file_path, encoding="utf-8") as f:
+                    packages = json.load(f)
+            except (json.JSONDecodeError, UnicodeDecodeError):
+                continue
+
+            if packages:
+                push_commit = idx == len(json_files)  # only True on last
+                cargo.collect_packages_from_cargo(packages, self.vcs_response, push_commit)

minecode_pipeline/pipes/__init__.py

@@ -0,0 +1,52 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# purldb is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+import textwrap
+from pathlib import Path
+import saneyaml
+from aboutcode import hashid
+
+ALLOWED_HOST = "ALLOWED_HOST"
+VERSION = "ALLOWED_HOST"
+author_name = "FEDERATEDCODE_GIT_SERVICE_NAME"
+author_email = "FEDERATEDCODE_GIT_SERVICE_EMAIL"
+remote_name = "origin"
+
+
+def write_purls_to_repo(repo, package, packages_yaml, push_commit=False):
+    """Write or update package purls in the repo and optionally commit/push changes."""
+
+    ppath = hashid.get_package_purls_yml_file_path(package)
+    add_purl_result(packages_yaml, repo, ppath)
+
+    if push_commit:
+        change_type = "Add" if ppath in repo.untracked_files else "Update"
+        commit_message = f"""\
+        {change_type} list of available {package} versions
+        Tool: pkg:github/aboutcode-org/purldb@v{VERSION}
+        Reference: https://{ALLOWED_HOST}/
+        Signed-off-by: {author_name} <{author_email}>
+        """
+
+        default_branch = repo.active_branch.name
+        repo.index.commit(textwrap.dedent(commit_message))
+        repo.git.push(remote_name, default_branch, "--no-verify")
+
+
+def add_purl_result(purls, repo, purls_file):
+    """Add package urls result to the local Git repository."""
+    relative_purl_file_path = Path(*purls_file.parts[1:])
+
+    write_to = Path(repo.working_dir) / relative_purl_file_path
+    write_to.parent.mkdir(parents=True, exist_ok=True)
+
+    with open(purls_file, encoding="utf-8", mode="w") as f:
+        f.write(saneyaml.dump(purls))
+
+    repo.index.add([relative_purl_file_path])
+    return relative_purl_file_path

minecode_pipeline/pipes/cargo.py

@@ -0,0 +1,26 @@
+from packageurl import PackageURL
+from aboutcode.hashid import get_core_purl
+from minecode_pipeline.pipes import write_purls_to_repo
+
+
+def collect_packages_from_cargo(packages, repo, push_commit=False):
+    """Collect Cargo package versions into purls and write them to the repo."""
+
+    if not packages and len(packages) > 0:
+        raise ValueError("No packages found")
+
+    updated_purls = []
+    first_pkg = packages[0]
+    version = first_pkg.get("vers")
+    name = first_pkg.get("name")
+    purl = PackageURL(type="cargo", name=name, version=version)
+    base_purl = get_core_purl(purl)
+
+    for package in packages:
+        version = package.get("vers")
+        name = package.get("name")
+
+        purl = PackageURL(type="cargo", name=name, version=version).to_string()
+        updated_purls.append(purl)
+
+    write_purls_to_repo(repo, base_purl, packages, push_commit)

minecode_pipeline/tests/__init__.py

@@ -0,0 +1,8 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# purldb is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#

setup.cfg

@@ -100,3 +100,4 @@ console_scripts =
 scancodeio_pipelines =
     matching = matchcode_pipeline.pipelines.matching:Matching
     d2d = scanpipe.pipelines.deploy_to_develop:DeployToDevelop
+    mine_cargo = minecode_pipeline.pipelines.mine_cargo:MineCargo