Merge pull request #1208 from dadokkio/develop

ikelos · web-flow · commit 3f5fce31e7d4 · 2024-09-02T21:51:50.000+01:00
yara-x support for yarascan
diff --git a/.github/workflows/build-pypi.yml b/.github/workflows/build-pypi.yml
@@ -18,7 +18,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: ["3.7"]
+        python-version: ["3.8"]
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
diff --git a/.github/workflows/install.yml b/.github/workflows/install.yml
@@ -8,7 +8,7 @@ jobs:
       fail-fast: false
       matrix:
         host: [ ubuntu-latest, windows-latest ]
-        python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ]
+        python-version: [ "3.8", "3.9", "3.10", "3.11" ]
     steps:
     - uses: actions/checkout@v4
 
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -6,7 +6,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: ["3.7"]
+        python-version: ["3.8"]
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
diff --git a/README.md b/README.md
@@ -20,7 +20,7 @@ more details.
 
 ## Requirements
 
-Volatility 3 requires Python 3.7.3 or later. To install the most minimal set of dependencies (some plugins will not work) use a command such as:
+Volatility 3 requires Python 3.8.0 or later. To install the most minimal set of dependencies (some plugins will not work) use a command such as:
 
 ```shell
 pip3 install -r requirements-minimal.txt
diff --git a/doc/requirements.txt b/doc/requirements.txt
@@ -4,5 +4,6 @@ sphinx_autodoc_typehints>=1.4.0
 sphinx-rtd-theme>=0.4.3
 
 yara-python
+yara-x
 pycryptodome
 pefile
diff --git a/pyproject.toml b/pyproject.toml
@@ -6,7 +6,7 @@ readme = "README.md"
 authors = [
     { name = "Volatility Foundation", email = "volatility@volatilityfoundation.org" },
 ]
-requires-python = ">=3.7.3"
+requires-python = ">=3.8.0"
 license = { text = "VSL" }
 dynamic = ["dependencies", "optional-dependencies", "version"]
 
diff --git a/test/requirements-testing.txt b/test/requirements-testing.txt
@@ -6,5 +6,6 @@ pefile>=2017.8.1 #foo
 
 # This is required for the yara plugins
 yara-python>=3.8.0
+yara-x>=0.5.0
 
 pytest>=7.0.0
diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py
@@ -7,7 +7,7 @@
 import sys
 import zipfile
 
-required_python_version = (3, 7, 3)
+required_python_version = (3, 8, 0)
 if (
     sys.version_info.major != required_python_version[0]
     or sys.version_info.minor < required_python_version[1]
diff --git a/volatility3/framework/plugins/linux/vmayarascan.py b/volatility3/framework/plugins/linux/vmayarascan.py
@@ -31,10 +31,10 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]
                 name="pslist", plugin=pslist.PsList, version=(2, 0, 0)
             ),
             requirements.PluginRequirement(
-                name="yarascan", plugin=yarascan.YaraScan, version=(1, 2, 0)
+                name="yarascan", plugin=yarascan.YaraScan, version=(2, 0, 0)
             ),
             requirements.VersionRequirement(
-                name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0)
+                name="yarascanner", component=yarascan.YaraScanner, version=(2, 1, 0)
             ),
             requirements.ModuleRequirement(
                 name="kernel",
diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py
@@ -29,7 +29,7 @@ def get_requirements(cls):
                 architectures=["Intel32", "Intel64"],
             ),
             requirements.VersionRequirement(
-                name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0)
+                name="yarascanner", component=yarascan.YaraScanner, version=(2, 1, 0)
             ),
         ]
 
@@ -38,7 +38,7 @@ def _generator(self):
 
         # Yara Rule to scan for MFT Header Signatures
         rules = yarascan.YaraScan.process_yara_options(
-            {"yara_rules": "/FILE0|FILE\\*|BAAD/"}
+            {"yara_string": "/FILE0|FILE\\*|BAAD/"}
         )
 
         # Read in the Symbol File
@@ -197,7 +197,7 @@ def _generator(self):
 
         # Yara Rule to scan for MFT Header Signatures
         rules = yarascan.YaraScan.process_yara_options(
-            {"yara_rules": "/FILE0|FILE\\*|BAAD/"}
+            {"yara_string": "/FILE0|FILE\\*|BAAD/"}
         )
 
         # Read in the Symbol File
diff --git a/volatility3/framework/plugins/windows/vadyarascan.py b/volatility3/framework/plugins/windows/vadyarascan.py
@@ -33,7 +33,7 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]
                 name="pslist", plugin=pslist.PsList, version=(2, 0, 0)
             ),
             requirements.PluginRequirement(
-                name="yarascan", plugin=yarascan.YaraScan, version=(1, 3, 0)
+                name="yarascan", plugin=yarascan.YaraScan, version=(2, 0, 0)
             ),
             requirements.ListRequirement(
                 name="pid",
@@ -73,26 +73,42 @@ def _generator(self):
                     )
                     continue
 
-                for match in rules.match(data=layer.read(start, size, True)):
-                    if yarascan.YaraScan.yara_returns_instances():
-                        for match_string in match.strings:
-                            for instance in match_string.instances:
+                data = layer.read(start, size, True)
+                if not yarascan.YaraScan._yara_x:
+                    for match in rules.match(data=data):
+                        if yarascan.YaraScan.yara_returns_instances():
+                            for match_string in match.strings:
+                                for instance in match_string.instances:
+                                    yield 0, (
+                                        format_hints.Hex(instance.offset + start),
+                                        task.UniqueProcessId,
+                                        match.rule,
+                                        match_string.identifier,
+                                        instance.matched_data,
+                                    )
+                        else:
+                            for offset, name, value in match.strings:
                                 yield 0, (
-                                    format_hints.Hex(instance.offset + start),
+                                    format_hints.Hex(offset + start),
                                     task.UniqueProcessId,
                                     match.rule,
+                                    name,
+                                    value,
+                                )
+                else:
+                    for match in rules.scan(data).matching_rules:
+                        for match_string in match.patterns:
+                            for instance in match_string.matches:
+                                yield 0, (
+                                    format_hints.Hex(instance.offset + start),
+                                    task.UniqueProcessId,
+                                    f"{match.namespace}.{match.identifier}",
                                     match_string.identifier,
-                                    instance.matched_data,
+                                    data[
+                                        instance.offset : instance.offset
+                                        + instance.length
+                                    ],
                                 )
-                    else:
-                        for offset, name, value in match.strings:
-                            yield 0, (
-                                format_hints.Hex(offset + start),
-                                task.UniqueProcessId,
-                                match.rule,
-                                name,
-                                value,
-                            )
 
     @staticmethod
     def get_vad_maps(
diff --git a/volatility3/framework/plugins/yarascan.py b/volatility3/framework/plugins/yarascan.py
@@ -13,58 +13,101 @@
 
 vollog = logging.getLogger(__name__)
 
+USE_YARA_X = False
+
 try:
-    import yara
+    import yara_x
+
+    USE_YARA_X = True
 
-    if tuple([int(x) for x in yara.__version__.split(".")]) < (3, 8):
-        raise ImportError
 except ImportError:
-    vollog.info(
-        "Python Yara (>3.8.0) module not found, plugin (and dependent plugins) not available"
-    )
-    raise
+    try:
+        import yara
+
+        if tuple(int(x) for x in yara.__version__.split(".")) < (3, 8):
+            raise ImportError
+
+        vollog.debug("Using yara-python module")
+
+    except ImportError:
+        vollog.info(
+            "Neither yara-x nor yara-python (>3.8.0) module not found, plugin (and dependent plugins) not available"
+        )
+        raise
 
 
 class YaraScanner(interfaces.layers.ScannerInterface):
-    _version = (2, 0, 0)
+    _version = (2, 1, 0)
 
     # yara.Rules isn't exposed, so we can't type this properly
     def __init__(self, rules) -> None:
         super().__init__()
         if rules is None:
             raise ValueError("No rules provided to YaraScanner")
         self._rules = rules
-        self.st_object = not tuple([int(x) for x in yara.__version__.split(".")]) < (
-            4,
-            3,
+        self.st_object = (
+            None
+            if USE_YARA_X
+            else not tuple(int(x) for x in yara.__version__.split(".")) < (4, 3)
         )
 
     def __call__(
         self, data: bytes, data_offset: int
     ) -> Iterable[Tuple[int, str, str, bytes]]:
-        for match in self._rules.match(data=data):
-            if YaraScan.yara_returns_instances():
-                for match_string in match.strings:
-                    for instance in match_string.instances:
+        if USE_YARA_X:
+            for match in self._rules.scan(data).matching_rules:
+                for match_string in match.patterns:
+                    for instance in match_string.matches:
                         yield (
                             instance.offset + data_offset,
-                            match.rule,
+                            f"{match.namespace}.{match.identifier}",
                             match_string.identifier,
-                            instance.matched_data,
+                            data[instance.offset : instance.offset + instance.length],
                         )
-            else:
-                for offset, name, value in match.strings:
-                    yield (offset + data_offset, match.rule, name, value)
+        else:
+            for match in self._rules.match(data=data):
+                if YaraScan.yara_returns_instances():
+                    for match_string in match.strings:
+                        for instance in match_string.instances:
+                            yield (
+                                instance.offset + data_offset,
+                                match.rule,
+                                match_string.identifier,
+                                instance.matched_data,
+                            )
+                else:
+                    for offset, name, value in match.strings:
+                        yield (offset + data_offset, match.rule, name, value)
+
+    @staticmethod
+    def get_rule(rule):
+        if USE_YARA_X:
+            return yara_x.compile(f"rule r1 {{strings: $a = {rule} condition: $a}}")
+        return yara.compile(
+            sources={"n": f"rule r1 {{strings: $a = {rule} condition: $a}}"}
+        )
+
+    @staticmethod
+    def from_compiled_file(filepath):
+        with resources.ResourceAccessor().open(filepath, "rb") as fp:
+            if USE_YARA_X:
+                return yara_x.Rules.deserialize_from(file=fp)
+            return yara.load(file=fp)
+
+    @staticmethod
+    def from_file(filepath):
+        with resources.ResourceAccessor().open(filepath, "rb") as fp:
+            if USE_YARA_X:
+                return yara_x.compile(fp.read().decode())
+            return yara.compile(file=fp)
 
 
 class YaraScan(plugins.PluginInterface):
     """Scans kernel memory using yara rules (string or file)."""
 
     _required_framework_version = (2, 0, 0)
-    _version = (1, 3, 0)
-
-    # TODO: When the major version is bumped, take the opportunity to rename the yara_rules config to yara_string
-    # or something that makes more sense
+    _version = (2, 0, 0)
+    _yara_x = USE_YARA_X
 
     @classmethod
     def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]:
@@ -99,10 +142,14 @@ def get_yarascan_option_requirements(
                 optional=True,
             ),
             requirements.StringRequirement(
-                name="yara_rules", description="Yara rules (as a string)", optional=True
+                name="yara_string",
+                description="Yara rules (as a string)",
+                optional=True,
             ),
             requirements.URIRequirement(
-                name="yara_file", description="Yara rules (as a file)", optional=True
+                name="yara_file",
+                description="Yara rules (as a file)",
+                optional=True,
             ),
             # This additional requirement is to follow suit with upstream, who feel that compiled rules could potentially be used to execute malicious code
             # As such, there's a separate option to run compiled files, as happened with yara-3.9 and later
@@ -121,38 +168,28 @@ def get_yarascan_option_requirements(
 
     @classmethod
     def yara_returns_instances(cls) -> bool:
-        st_object = not tuple([int(x) for x in yara.__version__.split(".")]) < (
-            4,
-            3,
-        )
-        return st_object
+        return not tuple(int(x) for x in yara.__version__.split(".")) < (4, 3)
 
     @classmethod
     def process_yara_options(cls, config: Dict[str, Any]):
         rules = None
-        if config.get("yara_rules", None) is not None:
-            rule = config["yara_rules"]
+        if config.get("yara_string") is not None:
+            rule = config["yara_string"]
             if rule[0] not in ["{", "/"]:
                 rule = f'"{rule}"'
             if config.get("case", False):
                 rule += " nocase"
             if config.get("wide", False):
                 rule += " wide ascii"
-            rules = yara.compile(
-                sources={"n": f"rule r1 {{strings: $a = {rule} condition: $a}}"}
-            )
-        elif config.get("yara_source", None) is not None:
-            rules = yara.compile(source=config["yara_source"])
-        elif config.get("yara_file", None) is not None:
-            rules = yara.compile(
-                file=resources.ResourceAccessor().open(config["yara_file"], "rb")
-            )
-        elif config.get("yara_compiled_file", None) is not None:
-            rules = yara.load(
-                file=resources.ResourceAccessor().open(
-                    config["yara_compiled_file"], "rb"
-                )
+            rules = YaraScanner.get_rule(rule)
+        elif config.get("yara_file") is not None:
+            vollog.debug(f"Plain file: {config['yara_file']} - yara-x: {USE_YARA_X}")
+            rules = YaraScanner.from_file(config["yara_file"])
+        elif config.get("yara_compiled_file") is not None:
+            vollog.debug(
+                f"Compiled file: {config['yara_compiled_file']} - yara-x: {USE_YARA_X}"
             )
+            rules = YaraScanner.from_compiled_file(config["yara_compiled_file"])
         else:
             vollog.error("No yara rules, nor yara rules file were specified")
         return rules

Original file line number	Diff line number	Diff line change
`@@ -6,7 +6,7 @@ readme = "README.md"`
`6`	`6`	`authors = [`
`7`	`7`	`{ name = "Volatility Foundation", email = "[email protected]" },`
`8`	`8`	`]`
`9`		`-requires-python = ">=3.7.3"`
	`9`	`+requires-python = ">=3.8.0"`
`10`	`10`	`license = { text = "VSL" }`
`11`	`11`	`dynamic = ["dependencies", "optional-dependencies", "version"]`
`12`	`12`