Small HTTP session fixes (#4601)

Author: gpotter2

scapy/layers/http.py

@@ -652,16 +652,7 @@ def tcp_reassemble(cls, data, metadata, _):
             is_response = isinstance(http_packet.payload, cls.clsresp)
             # Packets may have a Content-Length we must honnor
             length = http_packet.Content_Length
-            # Heuristic to try and detect instant HEAD responses, as those include a
-            # Content-Length that must not be honored. This is a bit crappy, and assumes
-            # that a 'HEAD' will never include an Encoding...
-            if (
-                is_response and
-                data.endswith(b"\r\n\r\n") and
-                not http_packet[HTTPResponse]._get_encodings()
-            ):
-                detect_end = lambda _: True
-            elif length is not None:
+            if length is not None:
                 # The packet provides a Content-Length attribute: let's
                 # use it. When the total size of the frags is high enough,
                 # we have the packet
@@ -672,8 +663,12 @@ def tcp_reassemble(cls, data, metadata, _):
                     detect_end = lambda dat: len(dat) - http_length >= length
                 else:
                     # The HTTP layer isn't fully received.
-                    detect_end = lambda dat: False
-                    metadata["detect_unknown"] = True
+                    if metadata.get("tcp_end", False):
+                        # This was likely a HEAD response. Ugh
+                        detect_end = lambda dat: True
+                    else:
+                        detect_end = lambda dat: False
+                        metadata["detect_unknown"] = True
             else:
                 # It's not Content-Length based. It could be chunked
                 encodings = http_packet[cls].payload._get_encodings()
@@ -833,7 +828,7 @@ def request(self, url, data=b"", timeout=5, follow_redirects=True, **headers):
         Perform a HTTP(s) request.
         """
         # Parse request url
-        m = re.match(r"(https?)://([^/:]+)(?:\:(\d+))?(?:/(.*))?", url)
+        m = re.match(r"(https?)://([^/:]+)(?:\:(\d+))?(/.*)?", url)
         if not m:
             raise ValueError("Bad URL !")
         transport, host, port, path = m.groups()

scapy/layers/l2.py

@@ -158,7 +158,11 @@ def getmacbyip(ip, chainCC=0):
     # Check the routing table
     iff, _, gw = conf.route.route(ip)
 
-    # Broadcast case
+    # Limited broadcast
+    if ip == "255.255.255.255":
+        return "ff:ff:ff:ff:ff:ff"
+
+    # Directed broadcast
     if (iff == conf.loopback_name) or (ip in conf.route.get_if_bcast(iff)):
         return "ff:ff:ff:ff:ff:ff"
 

scapy/sessions.py

@@ -12,7 +12,7 @@
 
 from scapy.compat import orb
 from scapy.config import conf
-from scapy.packet import NoPayload, Packet
+from scapy.packet import Packet
 from scapy.pton_ntop import inet_pton
 
 # Typing imports
@@ -310,8 +310,6 @@ def process(self,
         if TCP not in pkt:
             return pkt
         pay = pkt[TCP].payload
-        if isinstance(pay, (NoPayload, conf.padding_layer)):
-            return pkt
         new_data = pay.original
         # Match packets by a unique TCP identifier
         ident = self._get_ident(pkt)
@@ -333,16 +331,22 @@ def process(self,
             metadata["tcp_reassemble"] = tcp_reassemble = streamcls(pay_class)
         else:
             tcp_reassemble = metadata["tcp_reassemble"]
-        # Get a relative sequence number for a storage purpose
-        relative_seq = metadata.get("relative_seq", None)
-        if relative_seq is None:
-            relative_seq = metadata["relative_seq"] = seq - 1
-        seq = seq - relative_seq
-        # Add the data to the buffer
-        data.append(new_data, seq)
+
+        if pay:
+            # Get a relative sequence number for a storage purpose
+            relative_seq = metadata.get("relative_seq", None)
+            if relative_seq is None:
+                relative_seq = metadata["relative_seq"] = seq - 1
+            seq = seq - relative_seq
+            # Add the data to the buffer
+            data.append(new_data, seq)
+
         # Check TCP FIN or TCP RESET
         if pkt[TCP].flags.F or pkt[TCP].flags.R:
             metadata["tcp_end"] = True
+        elif not pay:
+            # If there's no payload and the stream isn't ending, ignore.
+            return pkt
 
         # In case any app layer protocol requires it,
         # allow the parser to inspect TCP PSH flag
@@ -393,7 +397,8 @@ def process(self,
             if isinstance(packet, conf.padding_layer):
                 return None
             # Rebuild resulting packet
-            pay.underlayer.remove_payload()
+            if pay:
+                pay.underlayer.remove_payload()
             if IP in pkt:
                 pkt[IP].len = None
                 pkt[IP].chksum = None

test/scapy/layers/http.uts

@@ -79,11 +79,11 @@ assert HTTPRequest in a[3]
 assert a[3].Method == b"HEAD"
 assert a[3].User_Agent == b'curl/7.88.1'
 
-assert HTTPResponse in a[5]
-assert a[5].Content_Type == b'text/html; charset=UTF-8'
-assert a[5].Expires == b'Mon, 01 Apr 2024 22:25:38 GMT'
-assert a[5].Reason_Phrase == b'Moved Permanently'
-assert a[5].X_Frame_Options == b"SAMEORIGIN"
+assert HTTPResponse in a[6]
+assert a[6].Content_Type == b'text/html; charset=UTF-8'
+assert a[6].Expires == b'Mon, 01 Apr 2024 22:25:38 GMT'
+assert a[6].Reason_Phrase == b'Moved Permanently'
+assert a[6].X_Frame_Options == b"SAMEORIGIN"
 
 = HTTP build with 'chunked' content type
 
@@ -214,7 +214,7 @@ filename = scapy_path("/test/pcaps/http_tcp_psh.pcap.gz")
 
 pkts = sniff(offline=filename, session=TCPSession)
 
-assert len(pkts) == 15
+assert len(pkts) == 14
 # Verify a split header exists in the packet
 assert pkts[5].User_Agent == b'example_user_agent'