misc addons

Author: jullrich

.gitignore

@@ -1 +1,106 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+# C extensions
+*.so
 *~
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# dotenv
+.env
+
+# virtualenv
+.venv
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+# Vim
+*.swn
+*.swm
+*.swo
+*.swp

README.md

@@ -5,9 +5,8 @@ See https://isc.sans.edu/diary.html?storyid=22900
 
 This is a simple (too simple?) Python script that will read a pcap, find HTTP requests and turn them into cURL commands for replay.
 
-Little effort is made to verify that the requests are valid. This is intended to extract well formed requests that were created by your browser. Not necessarily intedned for malicious requests. It also does not reassemble TCP streams (yet). Browsers typically send requests as one packet, but large requests will fail.
+Little effort is made to verify that the requests are valid. This is intended to extract well formed requests that were created by your browser. Not necessarily intended for malicious requests. It also does not reassemble TCP streams (yet). Browsers typically send requests as one packet, but large requests will fail.
 
-DISCLAIMER: I am not a Python coder. I do not like Python. I have to use it once in a while because I love Scapy.
+DISCLAIMER: I am not a Python coder. I do not like Python. I have to use it once in a while because I love [Scapy](http://www.secdev.org/projects/scapy/).
 
 CREDIT: Stackoverflow
-

TODO

@@ -1,6 +1,5 @@
 - session reassembly
 - make the matches for things like "Host" more forgiving, at least case insensitive
 - allow for HTTP traffic not on port 80
-- only allow valid "Methods"
 - better URL validation
 - Escape single quotes

pcap2curl.py

@@ -1,43 +1,64 @@
 #!/usr/bin/env python
 
 import sys
-from scapy.all import *
+from scapy.all import PcapReader, re, Raw, TCP
 
 
-if len(sys.argv) != 2:
-  print ("I need an input file. Usage ./pcap2curl.py inputfilename")
-  exit()
+VALID_METHODS = [
+    "GET",
+    "HEAD",
+    "POST",
+    "PUT",
+    "DELETE",
+    "CONNECT",
+    "OPTIONS",
+    "TRACE",
+    "PATCH"
+]  # see https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods
 
-infile = sys.argv[1]
-
-packets=rdpcap(infile)
 
 def payload2curl(p):
-   lines=re.compile("[\n\r]+").split(p)
-   startline=re.search('^([A-Z]+) ([^ ]+) (HTTP\/[0-9\/]+)',lines[0])
-   curl='curl ';
-   method=startline.group(1)
-   url=startline.group(2)
-   version=startline.group(3)
-
-   del lines[0]
-   headers=[]
-   for line in lines:
-       if ":" in line:
-         headers.append("-H '"+line+"'")
-       if "Host:" in line:
-         hostheader=re.search("^Host: (.*)",line)
-         hostname=hostheader.group(1)
-
-   if hostname not in url:
-     url='http://'+hostname+'/'+url
-   curl=curl+' '+"'"+url+"' \\\n"
-   curl=curl+'-X'+method+" \\\n"
-   curl=curl+" \\\n".join(headers)
-   return curl
-
-for p in packets:
-  payload=''
-  if p.haslayer(TCP) and p.haslayer(Raw) and  p[TCP].dport == 80:
-    payload=p[Raw].load
-    print (payload2curl(payload))
+    lines = re.compile("[\n\r]+").split(p.decode())
+    start_line = re.search("^([A-Z]+) ([^ ]+) (HTTP\/[0-9\/]+)", lines[0])
+    method = start_line.group(1)
+    url = start_line.group(2)
+    version = start_line.group(3)  # Never used
+
+    if method not in VALID_METHODS:
+        return
+
+    del lines[0]
+    headers = []
+    for line in lines:
+        if ":" in line:
+            headers.append("-H '{}'".format(line))
+        if "Host:" in line:
+            host_header = re.search("^Host: (.*)", line)
+            host_name = host_header.group(1)
+
+    proto_host = 'http://{}/'.format(host_name)
+    if not url.startswith(proto_host):
+        url = "{}{}".format(proto_host, url[1:] if url[0] == "/" else url)
+    curl = "curl '{}' \\\n -X {} \\\n ".format(url, method)
+    curl += " \\\n ".join(headers)
+    return curl
+
+
+def main():
+    if len(sys.argv) != 2:
+        print ("I need an input file. Usage ./pcap2curl.py inputfilename")
+        return
+
+    infile = sys.argv[1]
+
+    with PcapReader(infile) as packets:
+        for p in packets:
+            if p.haslayer(TCP) and p.haslayer(Raw) and p[TCP].dport == 80:
+                payload = p[Raw].load
+                cmd = payload2curl(payload)
+                if cmd:
+                    print(cmd)
+
+
+if __name__ == "__main__":
+    main()

setup.py

@@ -0,0 +1,18 @@
+from setuptools import setup
+
+setup(
+    name='pcap2curl',
+
+    version='0.1',
+
+    description='Extract HTTP requests from pcap and turn them into cURL',
+
+    url='https://github.com/jullrich/pcap2curl',
+
+    author='Johannes Ullrich',
+
+    license='GNU',
+
+    install_requires=['scapy']
+
+)