Add support to mine maven Package-URLs #660

minecode_pipelines/miners/__init__.py

@@ -6,23 +6,3 @@
 # See https://github.com/aboutcode-org/purldb for support or download.
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
-
-import os
-import saneyaml
-
-from pathlib import Path
-
-from aboutcode.hashid import PURLS_FILENAME
-
-
-def write_packageurls_to_file(repo, base_dir, packageurls):
-    purl_file_rel_path = os.path.join(base_dir, PURLS_FILENAME)
-    purl_file_full_path = Path(repo.working_dir) / purl_file_rel_path
-    write_data_to_file(path=purl_file_full_path, data=packageurls)
-    return purl_file_rel_path
-
-
-def write_data_to_file(path, data):
-    path.parent.mkdir(parents=True, exist_ok=True)
-    with open(path, encoding="utf-8", mode="w") as f:
-        f.write(saneyaml.dump(data))

minecode_pipelines/pipelines/mine_maven.py

@@ -0,0 +1,53 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# http://nexb.com and https://github.com/aboutcode-org/scancode.io
+# The ScanCode.io software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode.io is provided as-is without warranties.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+# ScanCode.io should be considered or used as legal advice. Consult an Attorney
+# for any legal advice.
+#
+# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
+# Visit https://github.com/aboutcode-org/scancode.io for support and download.
+
+
+from minecode_pipelines import pipes
+from minecode_pipelines.pipes import maven
+from scanpipe.pipelines.publish_to_federatedcode import PublishToFederatedCode
+
+
+class MineMaven(PublishToFederatedCode):
+    """
+    Create DiscoveredPackages for packages found on maven:
+    - input: url of maven repo
+    - process index
+      - collect purls, grouped by package
+      - write to files
+      - publish to fetchcode
+      - loop
+
+    """
+
+    @classmethod
+    def steps(cls):
+        return (
+            cls.check_federatedcode_eligibility,
+            cls.collect_packages_from_maven,
+            cls.delete_cloned_repos,
+        )
+
+    def collect_packages_from_maven(self):
+        self.repos = maven.collect_packages_from_maven(self.project, self.log)
+
+    def delete_cloned_repos(self):
+        pipes.delete_cloned_repos(repos=self.repos, logger=self.log)

minecode_pipelines/pipes/__init__.py

@@ -7,15 +7,14 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
-import os
 import json
+import os
+from pathlib import Path
+
 import requests
 import saneyaml
 
-from pathlib import Path
-
 from aboutcode.hashid import PURLS_FILENAME
-
 from scanpipe.pipes.federatedcode import delete_local_clone
 from scanpipe.pipes.federatedcode import commit_and_push_changes
 
@@ -41,6 +40,13 @@ def fetch_checkpoint_from_github(config_repo, checkpoint_path):
     return checkpoint_data
 
 
+def get_checkpoint_from_file(cloned_repo, path):
+    checkpoint_path = os.path.join(cloned_repo.working_dir, path)
+    with open(checkpoint_path) as f:
+        checkpoint_data = json.load(f)
+    return checkpoint_data or {}
+
+
 def update_checkpoints_in_github(checkpoint, cloned_repo, path):
     checkpoint_path = os.path.join(cloned_repo.working_dir, path)
     write_data_to_json_file(path=checkpoint_path, data=checkpoint)

minecode_pipelines/pipes/java_stream.py

@@ -0,0 +1,51 @@
+# The MIT License (MIT)
+#
+# Copyright (c) 2014 Gustav Arngården
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+
+"""Reading from Java DataInputStream format."""
+
+import struct
+
+
+class DataInputStream:
+    def __init__(self, stream):
+        self.stream = stream
+
+    def read(self, n=1):
+        data = self.stream.read(n)
+        if len(data) != n:
+            # this is a problem but in most cases we have reached EOF
+            raise EOFError
+        return data
+
+    def read_byte(self):
+        return struct.unpack("b", self.read(1))[0]
+
+    def read_long(self):
+        return struct.unpack(">q", self.read(8))[0]
+
+    def read_utf(self):
+        utf_length = struct.unpack(">H", self.read(2))[0]
+        return self.read(utf_length)
+
+    def read_int(self):
+        return struct.unpack(">i", self.read(4))[0]

minecode_pipelines/pipes/java_stream.py.ABOUT

@@ -0,0 +1,14 @@
+about_resource: java_stream.py
+name: java_stream.py
+version: 7d118ceef9746981e6bc198861125ca2bb6f920f
+homepage_url: https://github.com/arngarden/python_java_datastream
+owner: Gustav Arngården
+copyright: Copyright (c) 2014 Gustav Arngården
+download_url: https://raw.githubusercontent.com/arngarden/python_java_datastream/7d118ceef9746981e6bc198861125ca2bb6f920f/data_input_stream.py
+license_text_file: license_expfession: mit
+licenses:
+  - key: mit
+    file: java_stream.py.LICENSE
+
+vcs_tool: git
+vcs_repo: https://github.com/arngarden/python_java_datastream

minecode_pipelines/pipes/java_stream.py.LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Gustav Arngården
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.