Merge pull request #1115 from stratosphereips/alya/fix_detecting_the_gw_in_pcaps

Detect GW ip and mac when analysing files by using the dst mac of outgoing traffic

Author: AlyaGomaa

docs/contributing.md

@@ -161,8 +161,14 @@ Evidence Handler is the only process that stops but keeps waiting in memory for
 Once all modules are done processing, EvidenceHandler is killed by the Process manager.
 
 
-### How the tests work?
+### How does the tests work?
 
 - Running the tests locally should be done using ./tests/run_all_tests.sh
 - It runs the unit tests first, then the integration tests.
 - Please get familiar with pytest first https://docs.pytest.org/en/stable/how-to/output.html
+
+### Where and how do we get the GW info?
+
+Using one of these 3 ways
+
+<img src="https://raw.githubusercontent.com/stratosphereips/StratosphereLinuxIPS/develop/docs/images/gw_info.jpg"

modules/ip_info/ip_info.py

@@ -1,5 +1,8 @@
 import platform
-from typing import Union
+from typing import (
+    Union,
+    Optional,
+)
 from uuid import uuid4
 import datetime
 import maxminddb
@@ -58,7 +61,6 @@ def init(self):
             "new_dns": self.c3,
             "check_jarm_hash": self.c4,
         }
-        self.is_gw_mac_set = False
         self.whitelist = Whitelist(self.logger, self.db)
         self.is_running_non_stop: bool = self.db.is_running_non_stop()
         self.valid_tlds = whois.validTlds()
@@ -307,7 +309,7 @@ async def shutdown_gracefully(self):
         await self.reading_mac_db_task
 
     # GW
-    def get_gateway_ip(self):
+    def get_gateway_ip_if_interface(self):
         """
         Slips tries different ways to get the ip of the default gateway
         this method tries to get the default gateway IP address using ip route
@@ -338,7 +340,7 @@ def get_gateway_ip(self):
             gw_ip = route_default_result[2]
         return gw_ip
 
-    def get_gateway_mac(self, gw_ip: str):
+    def get_gateway_mac(self, gw_ip: str) -> Optional[str]:
         """
         Given the gw_ip, this function tries to get the MAC
          from arp.log or from arp tables
@@ -352,6 +354,7 @@ def get_gateway_mac(self, gw_ip: str):
             return gw_mac
 
         if not self.is_running_non_stop:
+            # running on pcap or a given zeek file/dir
             # no MAC in arp.log (in the db) and can't use arp tables,
             # so it's up to the db.is_gw_mac() function to determine the gw mac
             # if it's seen associated with a public IP
@@ -492,9 +495,14 @@ def pre_main(self):
         utils.drop_root_privs()
         self.wait_for_dbs()
         # the following method only works when running on an interface
-        if ip := self.get_gateway_ip():
+        if ip := self.get_gateway_ip_if_interface():
             self.db.set_default_gateway("IP", ip)
 
+            # whether we found the gw ip using dhcp in profiler
+            # or using ip route using self.get_gateway_ip()
+            # now that it's found, get and store the mac addr of it
+            self.get_gateway_mac(ip)
+
     def handle_new_ip(self, ip: str):
         try:
             # make sure its a valid ip
@@ -531,16 +539,6 @@ async def main(self):
 
             self.get_vendor(mac_addr, profileid)
             self.check_if_we_have_pending_offline_mac_queries()
-            # set the gw mac and ip if they're not set yet
-            if not self.is_gw_mac_set:
-                # whether we found the gw ip using dhcp in profiler
-                # or using ip route using self.get_gateway_ip()
-                # now that it's found, get and store the mac addr of it
-                if ip := self.db.get_gateway_ip():
-                    # now that we know the GW IP address,
-                    # try to get the MAC of this IP (of the gw)
-                    self.get_gateway_mac(ip)
-                    self.is_gw_mac_set = True
 
         if msg := self.get_msg("new_dns"):
             msg = json.loads(msg["data"])

slips_files/core/database/redis_db/database.py

@@ -1539,7 +1539,10 @@ def increment_processed_flows(self):
         return self.r.incr(self.constants.PROCESSED_FLOWS, 1)
 
     def get_processed_flows_so_far(self) -> int:
-        return int(self.r.get(self.constants.PROCESSED_FLOWS))
+        processed_flows = self.r.get(self.constants.PROCESSED_FLOWS)
+        if not processed_flows:
+            return 0
+        return int(processed_flows)
 
     def store_std_file(self, **kwargs):
         """

slips_files/core/helpers/flow_handler.py

@@ -60,6 +60,10 @@ def new_software(self, profileid, flow):
 
 
 class FlowHandler:
+    """
+    Each flow seen by slips will be a different instance of this class
+    """
+
     def __init__(self, db, symbol_handler, flow):
         self.db = db
         self.publisher = Publisher(self.db)
@@ -161,9 +165,18 @@ def handle_notice(self):
         self.db.add_out_notice(self.profileid, self.twid, self.flow)
 
         if "Gateway_addr_identified" in self.flow.note:
-            # get the gw addr form the msg
-            gw_addr = self.flow.msg.split(": ")[-1].strip()
-            self.db.set_default_gateway("IP", gw_addr)
+            # foirst check if the gw ip and mac are set by
+            # profiler.get_gateway_info() or ip_info module
+            gw_ip = False
+            if not self.db.get_gateway_ip():
+                # get the gw addr from the msg
+                gw_ip = self.flow.msg.split(": ")[-1].strip()
+                self.db.set_default_gateway("IP", gw_ip)
+
+            if not self.db.get_gateway_mac() and gw_ip:
+                gw_mac = self.db.get_mac_addr_from_profile(f"profile_{gw_ip}")
+                if gw_mac:
+                    self.db.set_default_gateway("MAC", gw_mac)
 
         self.db.add_altflow(self.flow, self.profileid, self.twid, "benign")
 

slips_files/core/profiler.py

@@ -4,6 +4,7 @@
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; either version 2
 # of the License, or (at your option) any later version.
+import json
 
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
@@ -22,6 +23,7 @@
 import multiprocessing
 from typing import (
     List,
+    Optional,
 )
 
 import validators
@@ -95,6 +97,8 @@ def init(
         # is set by this proc to tell input proc that we are done
         # processing and it can exit no issue
         self.is_profiler_done_event = is_profiler_done_event
+        self.gw_mac = None
+        self.gw_ip = None
 
     def read_configuration(self):
         conf = ConfigParser()
@@ -140,6 +144,87 @@ def get_rev_profile(self):
         )
         return rev_profileid, rev_twid
 
+    def get_gw_ip_using_gw_mac(self) -> Optional[str]:
+        """
+        gets the ip of the given mac from the db
+        prioritizes returning the ipv4. if not found, the function returns
+        the ipv6. or none if both are not found.
+        """
+        # the db returns a serialized list of IPs belonging to this mac
+        gw_ips: str = self.db.get_ip_of_mac(self.gw_mac)
+
+        if not gw_ips:
+            return
+
+        gw_ips: List[str] = json.loads(gw_ips)
+        # try to get the ipv4 if found in that list
+        for ip in gw_ips:
+            try:
+                ipaddress.IPv4Address(ip)
+                return ip
+            except ipaddress.AddressValueError:
+                continue
+
+        # all of them are ipv6, return the first
+        return gw_ips[0]
+
+    def is_gw_info_detected(self, info_type: str) -> bool:
+        """
+        checks own attributes and the db for the gw mac/ip
+        :param info_type: can be 'mac' or 'ip'
+        """
+        info_mapping = {
+            "mac": ("gw_mac", self.db.get_gateway_mac),
+            "ip": ("gw_ip", self.db.get_gateway_ip),
+        }
+
+        if info_type not in info_mapping:
+            raise ValueError(f"Unsupported info_type: {info_type}")
+
+        attr, check_db_method = info_mapping[info_type]
+
+        if getattr(self, attr):
+            # the reason we don't just check the db is we don't want a db
+            # call per each flow
+            return True
+
+        # did some other module manage to get it?
+        if info := check_db_method():
+            setattr(self, attr, info)
+            return True
+
+        return False
+
+    def get_gateway_info(self):
+        """
+        Gets the IP and MAC of the gateway and stores them in the db
+
+        usually the mac of the flow going from a private ip -> a
+        public ip is the mac of the GW
+        """
+        gw_mac_found: bool = self.is_gw_info_detected("mac")
+        if not gw_mac_found:
+            if utils.is_private_ip(
+                self.flow.saddr
+            ) and not utils.is_ignored_ip(self.flow.daddr):
+                self.gw_mac: str = self.flow.dmac
+                self.db.set_default_gateway("MAC", self.gw_mac)
+                self.print(
+                    f"MAC address of the gateway detected: "
+                    f"{green(self.gw_mac)}"
+                )
+                gw_mac_found = True
+
+        # we need the mac to be set to be able to find the ip using it
+        if not self.is_gw_info_detected("ip") and gw_mac_found:
+            self.gw_ip: Optional[str] = self.get_gw_ip_using_gw_mac()
+            if self.gw_ip:
+                self.db.set_default_gateway("IP", self.gw_ip)
+                self.print(
+                    f"IP address of the gateway detected: "
+                    f"{green(self.gw_ip)}"
+                )
+
     def add_flow_to_profile(self):
         """
         This is the main function that takes the columns of a flow
@@ -170,6 +255,8 @@ def add_flow_to_profile(self):
                 # software and weird.log flows are allowed to not have a daddr
                 return False
 
+        self.get_gateway_info()
+
         # Check if the flow is whitelisted and we should not process it
         if self.whitelist.is_whitelisted_flow(self.flow):
             return True

tests/test_flow_handler.py

@@ -280,12 +280,17 @@ def test_handle_notice(flow):
     flow.note = "Gateway_addr_identified: 192.168.1.1"
     flow.msg = "Gateway_addr_identified: 192.168.1.1"
 
+    flow_handler.db.get_gateway_ip.return_value = False
+    flow_handler.db.get_gateway_mac.return_value = False
+    flow_handler.db.get_mac_addr_from_profile.return_value = "xyz"
+
     flow_handler.handle_notice()
 
     flow_handler.db.add_out_notice.assert_called_with(
         flow_handler.profileid, flow_handler.twid, flow
     )
-    flow_handler.db.set_default_gateway.assert_called_with("IP", "192.168.1.1")
+    flow_handler.db.set_default_gateway.assert_any_call("IP", "192.168.1.1")
+    flow_handler.db.set_default_gateway.assert_any_call("MAC", "xyz")
     flow_handler.db.add_altflow.assert_called_with(
         flow, flow_handler.profileid, flow_handler.twid, "benign"
     )

tests/test_ip_info.py

@@ -421,7 +421,7 @@ def test_get_gateway_ip(
     mocker.patch("platform.system", return_value=platform_system)
     mocker.patch("subprocess.check_output", return_value=subprocess_output)
     mocker.patch("sys.argv", ["-i", "eth0"])
-    result = ip_info.get_gateway_ip()
+    result = ip_info.get_gateway_ip_if_interface()
     assert result == expected_ip