Merge pull request #2623 from nexB/2615-installedwindowsprogram-installed-files-paths

Collect InstalledWindowsProgram installed files #2615

Author: pombredanne

src/packagedcode/win_pe.py

@@ -233,6 +233,14 @@ class WindowsExecutable(models.Package):
         '.winmd',
         '.sys',
         '.tlb',
+        '.exe_*',
+        '.dll_*',
+        '.mui_*',
+        '.mun_*',
+        '.com_*',
+        '.winmd_*',
+        '.sys_*',
+        '.tlb_*',
     )
     filetypes = ('pe32', 'for ms windows',)
     mimetypes = ('application/x-dosexec',)

src/packagedcode/win_reg.py

@@ -211,6 +211,11 @@ def reg_parse(location):
     """
     for installed_program in report_installed_programs(location):
         yield installed_program
+    for installed_program in report_installed_programs(
+        location,
+        registry_path='\\Wow6432Node\\Microsoft\\Windows\\CurrentVersion\\Uninstall'
+    ):
+        yield installed_program
     for installed_dotnet in report_installed_dotnet_versions(location):
         yield installed_dotnet
 
@@ -222,22 +227,38 @@ def get_installed_packages(root_dir, is_container=True):
     """
     # These paths are relative to a Windows docker image layer root directory
     if is_container:
-        software_registry_locations = [
-            os.path.join(root_dir, 'Hives', 'Software_Delta'),
-            os.path.join(root_dir, 'Files', 'Windows', 'System32', 'config', 'SOFTWARE')
-        ]
-        # We are setting the root to be the `Files` directory, since this
-        # directory represents the root `C:\` drive of a Windows installation
-        root_dir = os.path.join(root_dir, 'Files')
+        hives_software_delta_loc = os.path.join(root_dir, 'Hives', 'Software_Delta')
+        files_software_loc = os.path.join(
+            root_dir,
+            'Files',
+            'Windows',
+            'System32',
+            'config',
+            'SOFTWARE'
+        )
+        utilityvm_software_loc = os.path.join(
+            root_dir,
+            'UtilityVM',
+            'Files',
+            'Windows',
+            'System32',
+            'config',
+            'SOFTWARE'
+        )
+        root_prefixes_by_software_registry_locations = {
+            hives_software_delta_loc: 'Files',
+            files_software_loc: 'Files',
+            utilityvm_software_loc: os.path.join('UtilityVM', 'Files')
+        }
     else:
         # TODO: Add support for virtual machines
         raise Exception('Unsuported file system type')
 
-    for software_registry_loc in software_registry_locations:
+    for software_registry_loc, root_prefix in root_prefixes_by_software_registry_locations.items():
         if not os.path.exists(software_registry_loc):
             continue
         for package in reg_parse(software_registry_loc):
-            package.populate_installed_files(root_dir)
+            package.populate_installed_files(root_dir, root_prefix=root_prefix)
             yield package
 
 
@@ -264,16 +285,27 @@ def create_absolute_installed_file_path(root_dir, file_path):
     return str(Path(root_dir).joinpath(file_path))
 
 
-def create_relative_file_path(file_path, root_dir):
+def create_relative_file_path(file_path, root_dir, root_prefix=''):
     """
     Return a subpath of `file_path` that is relative to `root_dir`
 
     >>> file_path = '/home/test/example/foo.txt'
     >>> root_dir = '/home/test/'
     >>> create_relative_file_path(file_path, root_dir)
     'example/foo.txt'
+
+    If there is a `root_prefix`, then it is prepended to the resulting
+    relative file path.
+
+    >>> file_path = '/home/test/example/foo.txt'
+    >>> root_dir = '/home/test/'
+    >>> create_relative_file_path(file_path, root_dir, 'prefix')
+    'prefix/example/foo.txt'
     """
-    return str(Path(file_path).relative_to(root_dir))
+    relative_file_path = str(Path(file_path).relative_to(root_dir))
+    if root_prefix:
+        return os.path.join(root_prefix, relative_file_path)
+    return relative_file_path
 
 
 @attr.s()
@@ -285,11 +317,19 @@ def recognize(cls, location):
         for installed in reg_parse(location):
             yield installed
 
-    def populate_installed_files(self, root_dir):
+    def populate_installed_files(self, root_dir, root_prefix=''):
         install_location = self.extra_data.get('install_location')
         if not install_location:
             return
 
+        if root_prefix:
+            # The rootfs location of a Docker image layer can be in a
+            # subdirectory of the layer directory (where `root_dir` is the path
+            # to the layer directory), so we append `root_prefix` (where prefix
+            # is relative to the file paths within the Docker image layer's
+            # rootfs files) to `root_dir`
+            root_dir = os.path.join(root_dir, root_prefix)
+
         absolute_install_location = create_absolute_installed_file_path(
             root_dir=root_dir,
             file_path=install_location
@@ -301,7 +341,11 @@ def populate_installed_files(self, root_dir):
         for root, _, files in os.walk(absolute_install_location):
             for file in files:
                 installed_file_location = os.path.join(root, file)
-                relative_installed_file_path = create_relative_file_path(installed_file_location, root_dir)
+                relative_installed_file_path = create_relative_file_path(
+                    file_path=installed_file_location,
+                    root_dir=root_dir,
+                    root_prefix=root_prefix
+                )
                 installed_files.append(relative_installed_file_path)
 
         known_program_files = self.extra_data.get('known_program_files', [])
@@ -310,7 +354,11 @@ def populate_installed_files(self, root_dir):
                 root_dir=root_dir,
                 file_path=known_program_file_path,
             )
-            relative_known_file_path = create_relative_file_path(known_program_file_location, root_dir)
+            relative_known_file_path = create_relative_file_path(
+                file_path=known_program_file_location,
+                root_dir=root_dir,
+                root_prefix=root_prefix
+            )
             if (not os.path.exists(known_program_file_location)
                     or relative_known_file_path in installed_files):
                 continue

tests/packagedcode/data/plugin/help.txt

@@ -273,6 +273,6 @@ Package: windows-update
 --------------------------------------------
 Package: winexe
   class: packagedcode.win_pe:WindowsExecutable
-  extensions: .exe, .dll, .mui, .mun, .com, .winmd, .sys, .tlb
+  extensions: .exe, .dll, .mui, .mun, .com, .winmd, .sys, .tlb, .exe_*, .dll_*, .mui_*, .mun_*, .com_*, .winmd_*, .sys_*, .tlb_*
   filetypes: pe32, for ms windows
 

tests/packagedcode/data/win_reg/get_installed_packages_docker/expected-results

@@ -1,76 +1,154 @@
-{
-  "type": "windows-program",
-  "namespace": null,
-  "name": "Test",
-  "version": "0.0.1",
-  "qualifiers": {},
-  "subpath": null,
-  "primary_language": null,
-  "description": null,
-  "release_date": null,
-  "parties": [
-    {
-      "type": "organization",
-      "role": "publisher",
-      "name": "Test Publisher",
-      "email": null,
-      "url": null
-    }
-  ],
-  "keywords": [],
-  "homepage_url": null,
-  "download_url": null,
-  "size": null,
-  "sha1": null,
-  "md5": null,
-  "sha256": null,
-  "sha512": null,
-  "bug_tracking_url": null,
-  "code_view_url": null,
-  "vcs_url": null,
-  "copyright": null,
-  "license_expression": null,
-  "declared_license": null,
-  "notice_text": null,
-  "root_path": null,
-  "dependencies": [],
-  "contains_source_code": null,
-  "source_packages": [],
-  "installed_files": [
-    {
-      "path": "Program Files/Test/Test.dat",
-      "size": 0,
-      "sha1": null,
-      "md5": null,
-      "sha256": null,
-      "sha512": null
+[
+  {
+    "type": "windows-program",
+    "namespace": null,
+    "name": "Test",
+    "version": "0.0.1",
+    "qualifiers": {},
+    "subpath": null,
+    "primary_language": null,
+    "description": null,
+    "release_date": null,
+    "parties": [
+      {
+        "type": "organization",
+        "role": "publisher",
+        "name": "Test Publisher",
+        "email": null,
+        "url": null
+      }
+    ],
+    "keywords": [],
+    "homepage_url": null,
+    "download_url": null,
+    "size": null,
+    "sha1": null,
+    "md5": null,
+    "sha256": null,
+    "sha512": null,
+    "bug_tracking_url": null,
+    "code_view_url": null,
+    "vcs_url": null,
+    "copyright": null,
+    "license_expression": null,
+    "declared_license": null,
+    "notice_text": null,
+    "root_path": null,
+    "dependencies": [],
+    "contains_source_code": null,
+    "source_packages": [],
+    "installed_files": [
+      {
+        "path": "Files/Program Files/Test/Test.dat",
+        "size": 0,
+        "sha1": null,
+        "md5": null,
+        "sha256": null,
+        "sha512": null
+      },
+      {
+        "path": "Files/Program Files/Test/Test.exe",
+        "size": 0,
+        "sha1": null,
+        "md5": null,
+        "sha256": null,
+        "sha512": null
+      },
+      {
+        "path": "Files/Program Files/Test/Uninstall.exe",
+        "size": 0,
+        "sha1": null,
+        "md5": null,
+        "sha256": null,
+        "sha512": null
+      }
+    ],
+    "extra_data": {
+      "install_location": "C:\\Program Files\\Test\\",
+      "known_program_files": [
+        "C:\\Program Files\\Test\\Test.exe",
+        "C:\\Program Files\\Test\\Uninstall.exe"
+      ]
     },
-    {
-      "path": "Program Files/Test/Test.exe",
-      "size": 0,
-      "sha1": null,
-      "md5": null,
-      "sha256": null,
-      "sha512": null
-    },
-    {
-      "path": "Program Files/Test/Uninstall.exe",
-      "size": 0,
-      "sha1": null,
-      "md5": null,
-      "sha256": null,
-      "sha512": null
-    }
-  ],
-  "extra_data": {
-    "install_location": "C:\\Program Files\\Test\\",
-    "known_program_files": [
-      "C:\\Program Files\\Test\\Test.exe",
-      "C:\\Program Files\\Test\\Uninstall.exe"
-    ]
+    "purl": "pkg:windows-program/[email protected]",
+    "repository_homepage_url": null,
+    "repository_download_url": null,
+    "api_data_url": null
   },
-  "purl": "pkg:windows-program/[email protected]",
-  "repository_homepage_url": null,
-  "repository_download_url": null,
-  "api_data_url": null
-}
+  {
+    "type": "windows-program",
+    "namespace": null,
+    "name": "Test2",
+    "version": "0.0.1",
+    "qualifiers": {},
+    "subpath": null,
+    "primary_language": null,
+    "description": null,
+    "release_date": null,
+    "parties": [
+      {
+        "type": "organization",
+        "role": "publisher",
+        "name": "Test Publisher",
+        "email": null,
+        "url": null
+      }
+    ],
+    "keywords": [],
+    "homepage_url": null,
+    "download_url": null,
+    "size": null,
+    "sha1": null,
+    "md5": null,
+    "sha256": null,
+    "sha512": null,
+    "bug_tracking_url": null,
+    "code_view_url": null,
+    "vcs_url": null,
+    "copyright": null,
+    "license_expression": null,
+    "declared_license": null,
+    "notice_text": null,
+    "root_path": null,
+    "dependencies": [],
+    "contains_source_code": null,
+    "source_packages": [],
+    "installed_files": [
+      {
+        "path": "Files/Program Files (x86)/Test2/Test2.dat",
+        "size": 0,
+        "sha1": null,
+        "md5": null,
+        "sha256": null,
+        "sha512": null
+      },
+      {
+        "path": "Files/Program Files (x86)/Test2/Test2.exe",
+        "size": 0,
+        "sha1": null,
+        "md5": null,
+        "sha256": null,
+        "sha512": null
+      },
+      {
+        "path": "Files/Program Files (x86)/Test2/Uninstall.exe",
+        "size": 0,
+        "sha1": null,
+        "md5": null,
+        "sha256": null,
+        "sha512": null
+      }
+    ],
+    "extra_data": {
+      "install_location": "C:\\Program Files (x86)\\Test2\\",
+      "known_program_files": [
+        "C:\\Program Files (x86)\\Test2\\Test2.exe",
+        "C:\\Program Files (x86)\\Test2\\Uninstall.exe"
+      ]
+    },
+    "purl": "pkg:windows-program/[email protected]",
+    "repository_homepage_url": null,
+    "repository_download_url": null,
+    "api_data_url": null
+  }
+]