Fix maven package assembly

Adds a fix for maven packacge assembly, combining MANIFEST.MF and
pom.xml files into a single package, instead of multiple top level
packages. Also fixes pom.properties file parsing.

Signed-off-by: Ayan Sinha Mahapatra <[email protected]>

Author: AyanSinhaMahapatra

src/packagedcode/__init__.py

@@ -101,11 +101,10 @@
 
     haxe.HaxelibJsonHandler,
 
-    jar_manifest.JavaJarManifestHandler,
-    jar_manifest.JavaOSGiManifestHandler,
-
     maven.MavenPomXmlHandler,
     maven.MavenPomPropertiesHandler,
+    maven.JavaJarManifestHandler,
+    maven.JavaOSGiManifestHandler,
 
     misc.AndroidAppArchiveHandler,
     misc.AndroidLibraryHandler,

src/packagedcode/jar_manifest.py

@@ -9,8 +9,7 @@
 
 import re
 
-from packagedcode import models
-from packagedcode.maven import parse_scm_connection
+from packagedcode.utils import VCS_URLS
 from packagedcode.utils import normalize_vcs_url
 
 """
@@ -23,48 +22,6 @@
 """
 
 
-class JavaJarManifestHandler(models.DatafileHandler):
-    datasource_id = 'java_jar_manifest'
-    path_patterns = ('*/META-INF/MANIFEST.MF',)
-    default_package_type = 'jar'
-    default_primary_language = 'Java'
-    description = 'Java JAR MANIFEST.MF'
-    documentation_url = 'https://docs.oracle.com/javase/tutorial/deployment/jar/manifestindex.html'
-
-    @classmethod
-    def parse(cls, location):
-        sections = parse_manifest(location)
-        if sections:
-            main_section = sections[0]
-            manifest = get_normalized_java_manifest_data(main_section)
-            if manifest:
-                yield models.PackageData(**manifest,)
-
-    @classmethod
-    def assign_package_to_resources(cls, package, resource, codebase, package_adder):
-        # we want to root of the jar, two levels up
-        parent = resource.parent(codebase)
-        if parent:
-            parent = resource.parent(codebase)
-
-        if parent:
-            models.DatafileHandler.assign_package_to_resources(
-                package,
-                resource=parent,
-                codebase=codebase,
-                package_adder=package_adder,
-            )
-
-
-class JavaOSGiManifestHandler(JavaJarManifestHandler):
-    datasource_id = 'java_osgi_manifest'
-    # This is an empty tuple to avoid getting back two packages
-    # essentially this is the same as JavaJarManifestHandler
-    path_patterns = ()
-    default_primary_language = 'Java'
-    description = 'Java OSGi MANIFEST.MF'
-    default_package_type = 'osgi'
-
 
 def parse_manifest(location):
     """
@@ -258,13 +215,8 @@ def dget(s):
         else:
             name = i_title or am_nm or ext_nm or nm
             descriptions = [s_title, i_title, nm]
-    
-    if package_type == 'maven':
-        datasource_id = JavaJarManifestHandler.datasource_id
-    elif package_type == 'jar':
-        datasource_id = JavaJarManifestHandler.datasource_id
-    elif package_type == 'osgi':
-        datasource_id = JavaOSGiManifestHandler.datasource_id
+
+    datasource_id =get_datasource_id(package_type=package_type)
 
     descriptions = unique(descriptions)
     descriptions = [d for d in descriptions if d and d.strip() and d != name]
@@ -391,6 +343,51 @@ def dget(s):
     return package
 
 
+def parse_scm_connection(scm_connection):
+    """
+    Return an SPDX vcs_url given a Maven `scm_connection` string or the string
+    as-is if it cannot be parsed.
+
+    See https://maven.apache.org/scm/scm-url-format.html
+        scm:<scm_provider><delimiter><provider_specific_part>
+
+    scm:git:git://server_name[:port]/path_to_repository
+    scm:git:http://server_name[:port]/path_to_repository
+    scm:git:https://server_name[:port]/path_to_repository
+    scm:git:ssh://server_name[:port]/path_to_repository
+    scm:git:file://[hostname]/path_to_repository
+    """
+
+    delimiter = '|' if '|' in scm_connection else ':'
+    segments = scm_connection.split(delimiter, 2)
+    if not len(segments) == 3:
+        # we cannot parse this so we return it as is
+        return scm_connection
+
+    _scm, scm_tool, vcs_url = segments
+    # TODO: vcs_tool is not yet supported
+    normalized = normalize_vcs_url(vcs_url, vcs_tool=scm_tool)
+    if normalized:
+        vcs_url = normalized
+
+    if not vcs_url.startswith(VCS_URLS):
+        if not vcs_url.startswith(scm_tool):
+            vcs_url = '{scm_tool}+{vcs_url}'.format(**locals())
+
+    return vcs_url
+
+
+def get_datasource_id(package_type):
+    from packagedcode.maven import JavaJarManifestHandler
+    from packagedcode.maven import JavaOSGiManifestHandler
+
+    if package_type == 'maven':
+        return JavaJarManifestHandler.datasource_id
+    elif package_type == 'jar':
+        return JavaJarManifestHandler.datasource_id
+    elif package_type == 'osgi':
+        return JavaOSGiManifestHandler.datasource_id
+
 def is_id(s):
     """
     Return True if `s` is some kind of id.

src/packagedcode/maven.py

@@ -13,15 +13,18 @@
 
 import javaproperties
 import lxml
+from fnmatch import fnmatchcase
 from packageurl import PackageURL
 from pymaven import artifact
 from pymaven import pom
 from pymaven.pom import strip_namespace
 
 from commoncode import fileutils
 from packagedcode import models
+from packagedcode.jar_manifest import parse_scm_connection
+from packagedcode.jar_manifest import parse_manifest
+from packagedcode.jar_manifest import get_normalized_java_manifest_data
 from packagedcode.utils import normalize_vcs_url
-from packagedcode.utils import VCS_URLS
 from textcode import analysis
 from typecode import contenttype
 
@@ -52,10 +55,117 @@
 there is no pom.properties check if there are side-by-side artifacts
 """
 
-# TODO: combine with jar_manifets.py
+class MavenBasePackageHandler(models.DatafileHandler):
+
+    @classmethod
+    def assemble(cls, package_data, resource, codebase, package_adder=models.add_to_package):
+        """
+        """
+        if codebase.has_single_resource:
+            yield from models.DatafileHandler.assemble(package_data, resource, codebase)
+            return
+
+        datafile_path = resource.path
+
+        # This order is important as we want pom.xml to be used for package
+        # creation and then to update from MANIFEST later 
+        manifest_path_pattern = '*/META-INF/MANIFEST.MF'
+        nested_pom_xml_path_pattern = '*/META-INF/maven/**/pom.xml'
+        datafile_name_patterns = (nested_pom_xml_path_pattern, manifest_path_pattern)
+
+        is_manifest = fnmatchcase(datafile_path, manifest_path_pattern)
+        is_pom_xml = fnmatchcase(datafile_path, nested_pom_xml_path_pattern)
+        if is_manifest:
+            meta_inf_resource = resource.parent(codebase)
+        elif is_pom_xml:
+            upward_segments = 4
+            root = resource
+            for _ in range(upward_segments):
+                root = root.parent(codebase)
+            meta_inf_resource = root
+        else:
+            yield from MavenPomXmlHandlerMixin.assemble(package_data, resource, codebase)
+            return
+
+        pom_xmls = []
+        manifests = []
+        for r in meta_inf_resource.walk(codebase):
+            if fnmatchcase(r.path, manifest_path_pattern):
+                manifests.append(r)
+            if fnmatchcase(r.path, nested_pom_xml_path_pattern):
+                pom_xmls.append(r)
+
+        if len(pom_xmls) > 1:
+            yield from MavenPomXmlHandlerMixin.assemble(package_data, resource, codebase)
+            return
+
+        if manifests and pom_xmls:
+            #raise Exception(resource.path, meta_inf_resource, datafile_name_patterns, package_adder)
+            parent_resource = meta_inf_resource.parent(codebase)
+            if not parent_resource:
+                parent_resource = meta_inf_resource
+            yield from cls.assemble_from_many_datafiles_in_directory(
+                    datafile_name_patterns=datafile_name_patterns,
+                    directory=meta_inf_resource,
+                    codebase=codebase,
+                    package_adder=package_adder,
+                    ignore_name_check=True,
+                    parent_resource=parent_resource,
+                )
+        elif manifests and not pom_xmls:
+            yield from JavaJarManifestHandlerMixin.assemble(package_data, resource, codebase)
+        elif pom_xmls and not manifests:
+            yield from MavenPomXmlHandlerMixin.assemble(package_data, resource, codebase)
+        else:
+            yield from models.DatafileHandler.assemble(package_data, resource, codebase)
 
 
-class MavenPomXmlHandler(models.DatafileHandler):
+class JavaJarManifestHandler(MavenBasePackageHandler):
+    datasource_id = 'java_jar_manifest'
+    path_patterns = ('*/META-INF/MANIFEST.MF',)
+    default_package_type = 'jar'
+    default_primary_language = 'Java'
+    description = 'Java JAR MANIFEST.MF'
+    documentation_url = 'https://docs.oracle.com/javase/tutorial/deployment/jar/manifestindex.html'
+
+    @classmethod
+    def parse(cls, location):
+        sections = parse_manifest(location)
+        if sections:
+            main_section = sections[0]
+            manifest = get_normalized_java_manifest_data(main_section)
+            if manifest:
+                yield models.PackageData(**manifest,)
+
+
+class JavaJarManifestHandlerMixin(models.DatafileHandler):
+
+    @classmethod
+    def assign_package_to_resources(cls, package, resource, codebase, package_adder):
+        # we want to root of the jar, two levels up
+        parent = resource.parent(codebase)
+        if parent:
+            parent = resource.parent(codebase)
+        if parent:
+            models.DatafileHandler.assign_package_to_resources(
+                package,
+                resource=parent,
+                codebase=codebase,
+                package_adder=package_adder,
+            )
+
+
+class JavaOSGiManifestHandler(JavaJarManifestHandler):
+    datasource_id = 'java_osgi_manifest'
+    # This is an empty tuple to avoid getting back two packages
+    # essentially this is the same as JavaJarManifestHandler
+    path_patterns = ()
+    default_primary_language = 'Java'
+    description = 'Java OSGi MANIFEST.MF'
+    default_package_type = 'osgi'
+
+
+class MavenPomXmlHandler(MavenBasePackageHandler):
     datasource_id = 'maven_pom'
     # NOTE: Maven 1.x used project.xml
     path_patterns = ('*.pom', '*pom.xml',)
@@ -64,8 +174,6 @@ class MavenPomXmlHandler(models.DatafileHandler):
     description = 'Apache Maven pom'
     documentation_url = 'https://maven.apache.org/pom.html'
 
-    # TODO: implment more sophistcaed assembly with META-INF/MANIFEST.MF and META-INF/LICENSE
-
     @classmethod
     def is_datafile(cls, location, filetypes=tuple()):
         """
@@ -106,6 +214,23 @@ def parse(cls, location, base_url='https://repo1.maven.org/maven2'):
         if package_data:
             yield package_data
 
+    @classmethod
+    def get_top_level_resources(cls, manifest_resource, codebase):
+        """
+        Yield Resources that are top-level based on a JAR's directory structure
+        """
+        if 'META-INF' in manifest_resource.path:
+            path_segments = manifest_resource.path.split('META-INF')
+            leading_segment = path_segments[0].strip()
+            meta_inf_dir_path = f'{leading_segment}/META-INF'
+            meta_inf_resource = codebase.get_resource(meta_inf_dir_path)
+            if meta_inf_resource:
+                yield meta_inf_resource
+                yield from meta_inf_resource.walk(codebase)
+
+
+class MavenPomXmlHandlerMixin(models.DatafileHandler):
+
     @classmethod
     def assign_package_to_resources(cls, package, resource, codebase, package_adder):
         """
@@ -164,20 +289,6 @@ def assign_package_to_resources(cls, package, resource, codebase, package_adder)
             package_adder=package_adder
         )
 
-    @classmethod
-    def get_top_level_resources(cls, manifest_resource, codebase):
-        """
-        Yield Resources that are top-level based on a JAR's directory structure
-        """
-        if 'META-INF' in manifest_resource.path:
-            path_segments = manifest_resource.path.split('META-INF')
-            leading_segment = path_segments[0].strip()
-            meta_inf_dir_path = f'{leading_segment}/META-INF'
-            meta_inf_resource = codebase.get_resource(meta_inf_dir_path)
-            if meta_inf_resource:
-                yield meta_inf_resource
-                yield from meta_inf_resource.walk(codebase)
-
 
 # TODO: assemble with its pom!!
 class MavenPomPropertiesHandler(models.NonAssemblableDatafileHandler):
@@ -201,8 +312,8 @@ def parse(cls, location):
             if properties:
                 yield models.PackageData(
                     datasource_id=cls.datasource_id,
-                    type=cls.package_type,
-                    primary_language=cls.primary_language,
+                    type=cls.default_package_type,
+                    primary_language=cls.default_primary_language,
                     extra_data=dict(pom_properties=properties)
                 )
 
@@ -1267,40 +1378,6 @@ def build_vcs_and_code_view_urls(scm):
     return dict(vcs_url=vcs_url, code_view_url=code_view_url,)
 
 
-def parse_scm_connection(scm_connection):
-    """
-    Return an SPDX vcs_url given a Maven `scm_connection` string or the string
-    as-is if it cannot be parsed.
-
-    See https://maven.apache.org/scm/scm-url-format.html
-        scm:<scm_provider><delimiter><provider_specific_part>
-
-    scm:git:git://server_name[:port]/path_to_repository
-    scm:git:http://server_name[:port]/path_to_repository
-    scm:git:https://server_name[:port]/path_to_repository
-    scm:git:ssh://server_name[:port]/path_to_repository
-    scm:git:file://[hostname]/path_to_repository
-    """
-
-    delimiter = '|' if '|' in scm_connection else ':'
-    segments = scm_connection.split(delimiter, 2)
-    if not len(segments) == 3:
-        # we cannot parse this so we return it as is
-        return scm_connection
-
-    _scm, scm_tool, vcs_url = segments
-    # TODO: vcs_tool is not yet supported
-    normalized = normalize_vcs_url(vcs_url, vcs_tool=scm_tool)
-    if normalized:
-        vcs_url = normalized
-
-    if not vcs_url.startswith(VCS_URLS):
-        if not vcs_url.startswith(scm_tool):
-            vcs_url = '{scm_tool}+{vcs_url}'.format(**locals())
-
-    return vcs_url
-
-
 def get_extension(packaging):
     """
     We only care for certain artifacts extension/packaging/classifier.