Merge branch 'volatilityfoundation:develop' into fix-issue-895

Author: 616c696365

.github/workflows/install.yml

@@ -0,0 +1,31 @@
+name: Install Volatility3 test
+on: [push, pull_request]
+jobs:
+
+  install_test:
+    runs-on: ${{ matrix.host }}
+    strategy:
+      fail-fast: false
+      matrix:
+        host: [ ubuntu-latest, windows-latest ]
+        python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ]
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Setup python-pip
+      run: python -m pip install --upgrade pip
+
+    - name: Install dependencies
+      run: |
+        pip install -r requirements.txt
+
+    - name: Install volatility3
+      run: pip install .
+
+    - name: Run volatility3
+      run: vol --help

doc/source/conf.py

@@ -27,7 +27,7 @@ def setup(app):
 
     source_dir = os.path.abspath(os.path.dirname(__file__))
     sphinx.ext.apidoc.main(
-        argv=["-e", "-M", "-f", "-T", "-o", source_dir, volatility_directory]
+        ["-e", "-M", "-f", "-T", "-o", source_dir, volatility_directory]
     )
 
     # Go through the volatility3.framework.plugins files and change them to volatility3.plugins

doc/source/using-as-a-library.rst

@@ -54,6 +54,12 @@ also be included, which can be found in `volatility3.constants.PLUGINS_PATH`.
         volatility3.plugins.__path__ = <new_plugin_path> + constants.PLUGINS_PATH
         failures = framework.import_files(volatility3.plugins, True)
 
+.. note::
+
+    Volatility uses the `volatility3.plugins` namespace for all plugins (including those in `volatility3.framework.plugins`).
+    Please ensure you only use `volatility3.plugins` and only ever import plugins from this namespace.
+    This ensures the ability of users to override core plugins without needing write access to the framework directory.
+
 Once the plugins have been imported, we can interrogate which plugins are available.  The
 :py:func:`~volatility3.framework.list_plugins` call will
 return a dictionary of plugin names and the plugin classes.

setup.py

@@ -12,14 +12,15 @@
 
 def get_install_requires():
     requirements = []
-    with open("requirements-minimal.txt", "r", encoding = "utf-8") as fh:
+    with open("requirements-minimal.txt", "r", encoding="utf-8") as fh:
         for line in fh.readlines():
             stripped_line = line.strip()
             if stripped_line == "" or stripped_line.startswith("#"):
                 continue
             requirements.append(stripped_line)
     return requirements
 
+
 setuptools.setup(
     name="volatility3",
     description="Memory forensics framework",
@@ -36,12 +37,12 @@ def get_install_requires():
         "Documentation": "https://volatility3.readthedocs.io/",
         "Source Code": "https://github.com/volatilityfoundation/volatility3",
     },
-    python_requires=">=3.7.0",
-    include_package_data=True,
-    exclude_package_data={"": ["development", "development.*"], "development": ["*"]},
     packages=setuptools.find_namespace_packages(
-        exclude=["development", "development.*"]
+        include=["volatility3", "volatility3.*"]
     ),
+    package_dir={"volatility3": "volatility3"},
+    python_requires=">=3.7.0",
+    include_package_data=True,
     entry_points={
         "console_scripts": [
             "vol = volatility3.cli:main",

volatility3/framework/constants/__init__.py

@@ -45,7 +45,7 @@
 # We use the SemVer 2.0.0 versioning scheme
 VERSION_MAJOR = 2  # Number of releases of the library with a breaking change
 VERSION_MINOR = 5  # Number of changes that only add to the interface
-VERSION_PATCH = 0  # Number of changes that do not change the interface
+VERSION_PATCH = 2  # Number of changes that do not change the interface
 VERSION_SUFFIX = ""
 
 # TODO: At version 2.0.0, remove the symbol_shift feature

volatility3/framework/constants/linux/__init__.py

@@ -279,3 +279,5 @@
     "bpf",
     "checkpoint_restore",
 )
+
+ELF_MAX_EXTRACTION_SIZE = 1024 * 1024 * 1024 * 4 - 1

volatility3/framework/layers/intel.py

@@ -111,6 +111,11 @@ def _page_is_valid(entry: int) -> bool:
         """Returns whether a particular page is valid based on its entry."""
         return bool(entry & 1)
 
+    @staticmethod
+    def _page_is_dirty(entry: int) -> bool:
+        """Returns whether a particular page is dirty based on its entry."""
+        return bool(entry & (1 << 6))
+
     def canonicalize(self, addr: int) -> int:
         """Canonicalizes an address by performing an appropiate sign extension on the higher addresses"""
         if self._bits_per_register <= self._maxvirtaddr:
@@ -259,6 +264,10 @@ def is_valid(self, offset: int, length: int = 1) -> bool:
         except exceptions.InvalidAddressException:
             return False
 
+    def is_dirty(self, offset: int) -> bool:
+        """Returns whether the page at offset is marked dirty"""
+        return self._page_is_dirty(self._translate_entry(offset)[0])
+
     def mapping(
         self, offset: int, length: int, ignore_errors: bool = False
     ) -> Iterable[Tuple[int, int, int, int, str]]:

volatility3/framework/layers/vmware.py

@@ -4,6 +4,7 @@
 import contextlib
 import logging
 import struct
+import os
 from typing import Any, Dict, List, Optional
 
 from volatility3.framework import constants, exceptions, interfaces
@@ -232,6 +233,11 @@ def stack(
             )
 
             if not vmss_success and not vmsn_success:
+                vmem_file_basename = os.path.basename(location)
+                example_vmss_file_basename = os.path.basename(vmss)
+                vollog.warning(
+                    f"No metadata file found alongside VMEM file. A VMSS or VMSN file may be required to correctly process a VMEM file. These should be placed in the same directory with the same file name, e.g. {vmem_file_basename} and {example_vmss_file_basename}.",
+                )
                 return None
             new_layer_name = context.layers.free_layer_name("VmwareLayer")
             context.config[

volatility3/framework/plugins/linux/elfs.py

@@ -4,20 +4,26 @@
 """A module containing a collection of plugins that produce data typically
 found in Linux's /proc file system."""
 
-from typing import List
+import logging
+from typing import List, Optional, Type
 
-from volatility3.framework import renderers, interfaces
+from volatility3.framework import constants, interfaces, renderers
 from volatility3.framework.configuration import requirements
 from volatility3.framework.interfaces import plugins
 from volatility3.framework.objects import utility
 from volatility3.framework.renderers import format_hints
+from volatility3.framework.symbols import intermed
+from volatility3.framework.symbols.linux.extensions import elf
 from volatility3.plugins.linux import pslist
 
+vollog = logging.getLogger(__name__)
+
 
 class Elfs(plugins.PluginInterface):
     """Lists all memory mapped ELF files for all processes."""
 
     _required_framework_version = (2, 0, 0)
+    _version = (2, 0, 0)
 
     @classmethod
     def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]:
@@ -36,9 +42,93 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]
                 element_type=int,
                 optional=True,
             ),
+            requirements.BooleanRequirement(
+                name="dump",
+                description="Extract listed processes",
+                default=False,
+                optional=True,
+            ),
         ]
 
+    @classmethod
+    def elf_dump(
+        cls,
+        context: interfaces.context.ContextInterface,
+        layer_name: str,
+        elf_table_name: str,
+        vma: interfaces.objects.ObjectInterface,
+        task: interfaces.objects.ObjectInterface,
+        open_method: Type[interfaces.plugins.FileHandlerInterface],
+    ) -> Optional[interfaces.plugins.FileHandlerInterface]:
+        """Extracts an ELF as a FileHandlerInterface
+        Args:
+            context: the context to operate upon
+            layer_name: The name of the layer on which to operate
+            elf_table_name: the name for the symbol table containing the symbols for ELF-files
+            vma: virtual memory allocation of ELF
+            task: the task object whose memory should be output
+            open_method: class to provide context manager for opening the file
+        Returns:
+            An open FileHandlerInterface object containing the complete data for the task or None in the case of failure
+        """
+
+        proc_layer = context.layers[layer_name]
+        file_handle = None
+
+        elf_object = context.object(
+            elf_table_name + constants.BANG + "Elf",
+            offset=vma.vm_start,
+            layer_name=layer_name,
+        )
+
+        if not elf_object.is_valid():
+            return None
+
+        sections = {}
+        # TODO: Apply more effort to reconstruct ELF, e.g.: https://github.com/enbarberis/core2ELF64 ?
+        for phdr in elf_object.get_program_headers():
+            if phdr.p_type != 1:  # PT_LOAD = 1
+                continue
+
+            start = phdr.p_vaddr
+            size = phdr.p_memsz
+            end = start + size
+
+            # Use complete memory pages for dumping
+            # If start isn't a multiple of 4096, stick to the highest multiple < start
+            # If end isn't a multiple of 4096, stick to the lowest multiple > end
+            if start % 4096:
+                start = start & ~0xFFF
+
+            if end % 4096:
+                end = (end & ~0xFFF) + 4096
+
+            real_size = end - start
+
+            # Check if ELF has a legitimate size
+            if real_size < 0 or real_size > constants.linux.ELF_MAX_EXTRACTION_SIZE:
+                raise ValueError(f"The claimed size of the ELF is invalid: {real_size}")
+
+            sections[start] = real_size
+
+        elf_data = b""
+        for section_start in sorted(sections.keys()):
+            read_size = sections[section_start]
+
+            buf = proc_layer.read(vma.vm_start + section_start, read_size, pad=True)
+            elf_data = elf_data + buf
+
+        file_handle = open_method(
+            f"pid.{task.pid}.{utility.array_to_string(task.comm)}.{vma.vm_start:#x}.dmp"
+        )
+        file_handle.write(elf_data)
+
+        return file_handle
+
     def _generator(self, tasks):
+        elf_table_name = intermed.IntermediateSymbolTable.create(
+            self.context, self.config_path, "linux", "elf", class_types=elf.class_types
+        )
         for task in tasks:
             proc_layer_name = task.add_process_layer()
             if not proc_layer_name:
@@ -60,6 +150,21 @@ def _generator(self, tasks):
 
                 path = vma.get_name(self.context, task)
 
+                file_output = "Disabled"
+                if self.config["dump"]:
+                    file_handle = self.elf_dump(
+                        self.context,
+                        proc_layer_name,
+                        elf_table_name,
+                        vma,
+                        task,
+                        self.open,
+                    )
+                    file_output = "Error outputting file"
+                    if file_handle:
+                        file_handle.close()
+                        file_output = str(file_handle.preferred_filename)
+
                 yield (
                     0,
                     (
@@ -68,6 +173,7 @@ def _generator(self, tasks):
                         format_hints.Hex(vma.vm_start),
                         format_hints.Hex(vma.vm_end),
                         path,
+                        file_output,
                     ),
                 )
 
@@ -81,6 +187,7 @@ def run(self):
                 ("Start", format_hints.Hex),
                 ("End", format_hints.Hex),
                 ("File Path", str),
+                ("File Output", str),
             ],
             self._generator(
                 pslist.PsList.list_tasks(

volatility3/framework/plugins/linux/malfind.py

@@ -3,14 +3,16 @@
 #
 
 from typing import List
-
+import logging
 from volatility3.framework import constants, interfaces
 from volatility3.framework import renderers
 from volatility3.framework.configuration import requirements
 from volatility3.framework.objects import utility
 from volatility3.framework.renderers import format_hints
 from volatility3.plugins.linux import pslist
 
+vollog = logging.getLogger(__name__)
+
 
 class Malfind(interfaces.plugins.PluginInterface):
     """Lists process memory ranges that potentially contain injected code."""
@@ -47,7 +49,14 @@ def _list_injections(self, task):
         proc_layer = self.context.layers[proc_layer_name]
 
         for vma in task.mm.get_vma_iter():
-            if vma.is_suspicious() and vma.get_name(self.context, task) != "[vdso]":
+            vma_name = vma.get_name(self.context, task)
+            vollog.debug(
+                f"Injections : processing PID {task.pid} : VMA {vma_name} : {hex(vma.vm_start)}-{hex(vma.vm_end)}"
+            )
+            if (
+                vma.is_suspicious(proc_layer)
+                and vma.get_name(self.context, task) != "[vdso]"
+            ):
                 data = proc_layer.read(vma.vm_start, 64, pad=True)
                 yield vma, data