Add Onyphe, Modat Generator & Fix generation issues

generate_all.sh

@@ -16,7 +16,8 @@ python3 generate-disposal.py
 #python3 generate-google.py > lists/google/list.json
 #python3 generate_majestic-million.py -n 10000
 #python3 generate-microsoft-azure.py
-python3 generate_mozilla_certificates.py
+# See https://github.com/MISP/misp-warninglists/issues/319
+# python3 generate_mozilla_certificates.py see 
 python3 generate_moz-top500.py
 python3 generate-office365.py
 python3 generate_phone_numbers.py
@@ -40,6 +41,8 @@ python3 generate-digitalside.py
 #python3 generate-gptbot.py
 python3 generate-cisco-umbrella-blockpage.py
 python3 generate-zscaler.py
+python3 generate-onyphe-scanner.py
+python3 generate-modat-scanner.py
 popd
 
 ./jq_all_the_things.sh

lists/modat-scanner/list.json

@@ -0,0 +1,85 @@
+{
+  "description": "Modat Scanner (https://www.modat.io/)",
+  "list": [
+    "104.236.209.11/32",
+    "134.209.173.54/32",
+    "141.95.206.110/32",
+    "142.91.102.186/31",
+    "148.113.189.33/32",
+    "148.113.190.57/32",
+    "148.113.193.57/32",
+    "148.113.193.59/32",
+    "148.113.193.79/32",
+    "148.113.206.49/32",
+    "148.113.208.45/32",
+    "148.113.210.228/32",
+    "148.113.210.254/32",
+    "148.113.211.131/32",
+    "148.113.212.55/32",
+    "148.113.213.23/32",
+    "148.113.213.24/32",
+    "148.113.214.202/32",
+    "148.113.214.204/32",
+    "148.113.214.206/32",
+    "148.113.214.212/32",
+    "148.113.214.99/32",
+    "148.113.216.105/32",
+    "148.113.221.195/32",
+    "148.113.221.28/32",
+    "15.235.224.227/32",
+    "15.235.224.238/31",
+    "15.235.225.83/32",
+    "15.235.226.164/32",
+    "15.235.226.26/32",
+    "15.235.227.163/32",
+    "15.235.227.39/32",
+    "15.235.227.46/32",
+    "15.235.227.52/32",
+    "15.235.227.54/32",
+    "15.235.227.64/32",
+    "15.235.236.76/32",
+    "15.235.236.87/32",
+    "158.69.125.92/32",
+    "162.19.80.39/32",
+    "162.19.80.7/32",
+    "212.73.148.0/28",
+    "212.73.148.16/30",
+    "212.73.148.20/31",
+    "212.73.148.22/32",
+    "217.182.213.55/32",
+    "217.182.213.59/32",
+    "31.14.32.4/30",
+    "31.14.32.8/32",
+    "46.105.70.190/32",
+    "51.161.172.223/32",
+    "51.161.174.170/32",
+    "51.222.152.150/32",
+    "51.68.54.105/32",
+    "51.68.54.55/32",
+    "51.68.54.67/32",
+    "54.37.81.238/32",
+    "54.38.193.48/32",
+    "57.129.64.10/32",
+    "57.129.64.219/32",
+    "57.129.64.237/32",
+    "85.217.140.0/28",
+    "85.217.140.16/31",
+    "85.217.149.0/28",
+    "85.217.149.16/29",
+    "85.217.149.24/30",
+    "91.134.73.222/32",
+    "91.134.73.230/32",
+    "91.134.84.178/32",
+    "91.134.9.78/32"
+  ],
+  "matching_attributes": [
+    "ip-src",
+    "ip-dst",
+    "domain|ip",
+    "ip-src|port",
+    "ip-dst|port"
+  ],
+  "name": "List of published IP address ranges for Modat Scanner",
+  "type": "cidr",
+  "version": 20251210
+}

lists/onyphe-scanner/list.json

@@ -0,0 +1,27 @@
+{
+  "description": "Onyphe Scanner (https://www.onyphe.io/)",
+  "list": [
+    "139.99.35.32/28",
+    "15.235.154.64/28",
+    "15.235.175.112/28",
+    "15.235.189.144/28",
+    "195.184.76.0/24",
+    "2001:41d0:33a:a00::400/124",
+    "2a14:f100::/29",
+    "51.254.49.96/28",
+    "91.196.152.0/24",
+    "91.230.168.0/24",
+    "91.231.89.0/24",
+    "94.231.206.0/24"
+  ],
+  "matching_attributes": [
+    "ip-src",
+    "ip-dst",
+    "domain|ip",
+    "ip-src|port",
+    "ip-dst|port"
+  ],
+  "name": "List of published IP address ranges for Onyphe Scanner",
+  "type": "cidr",
+  "version": 20251210
+}

requirements.txt

@@ -1,5 +1,6 @@
 beautifulsoup4
-pyOpenSSL==19.1.0
+# pyOpenSSL==19.1.0 - broke generate-crl-ip-domains.py
+pyopenssl<23.0.0
 python-dateutil==2.8.1
 requests
 dnspython

tools/generate-modat-scanner.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import json
+
+from generator import download_to_file, get_version, write_to_file, get_abspath_source_file, consolidate_networks
+
+def process(file, dst):
+    l = []
+    with open(get_abspath_source_file(file), 'r') as freetext_file:
+        for line in freetext_file:
+            if not line.startswith("#"):
+                cidr = line.rstrip()
+                l.append(cidr)
+
+    warninglist = {
+        'name': 'List of published IP address ranges for Modat Scanner',
+        'version': get_version(),
+        'description': 'Modat Scanner (https://www.modat.io/)',
+        'type': 'cidr',
+        'list': consolidate_networks(l),
+        'matching_attributes': ["ip-src", "ip-dst", "domain|ip", "ip-src|port", "ip-dst|port"]
+    }
+
+    write_to_file(warninglist, dst)
+
+
+if __name__ == '__main__':
+    modat_url = "https://scanner.modat.io/ipv4.txt"
+    modat_file = "modat-scanner.json"
+    modat_dst = "modat-scanner"
+
+    download_to_file(modat_url, modat_file) # Download in TMP
+    process(modat_file, modat_dst)

tools/generate-onyphe-scanner.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import json
+
+from generator import download_to_file, get_version, write_to_file, get_abspath_source_file, consolidate_networks
+
+def process(file, dst):
+    l = []
+    with open(get_abspath_source_file(file), 'r') as freetext_file:
+        for line in freetext_file:
+            if not line.startswith("#"):
+                cidr = line.rstrip()
+                l.append(cidr)
+
+    warninglist = {
+        'name': 'List of published IP address ranges for Onyphe Scanner',
+        'version': get_version(),
+        'description': 'Onyphe Scanner (https://www.onyphe.io/)',
+        'type': 'cidr',
+        'list': consolidate_networks(l),
+        'matching_attributes': ["ip-src", "ip-dst", "domain|ip", "ip-src|port", "ip-dst|port"]
+    }
+
+    write_to_file(warninglist, dst)
+
+
+if __name__ == '__main__':
+    onyphe_url = "https://www.onyphe.io/ip-ranges.txt"
+    onyphe_file = "onyphe-scanner.json"
+    onyphe_dst = "onyphe-scanner"
+
+    download_to_file(onyphe_url, onyphe_file) # Download in TMP
+    process(onyphe_file, onyphe_dst)

tools/generator.py

@@ -3,7 +3,7 @@
 import json
 import logging
 from inspect import currentframe, getframeinfo, getmodulename, stack
-from os import mkdir, path
+from os import mkdir, path, makedirs
 from typing import List, Union
 import gzip
 
@@ -128,6 +128,9 @@ def get_abspath_source_file(dst):
 
 
 def get_version():
+    '''
+        Return the YYYYMMDD as int for version.
+    '''
     return int(datetime.date.today().strftime('%Y%m%d'))
 
 
@@ -137,10 +140,13 @@ def unique_sorted_warninglist(warninglist):
 
 
 def write_to_file(warninglist, dst):
+    # Write the final Json file into the module folder name
     frame_records = stack()[1]
     caller = getmodulename(frame_records[1]).upper()
     try:
         warninglist = unique_sorted_warninglist(warninglist)
+        # Try to create the folder if it not exist yet
+        makedirs(path.dirname(get_abspath_list_file(dst)), exist_ok=True)
         with open(get_abspath_list_file(dst), 'w') as data_file:
             json.dump(warninglist, data_file, indent=2, sort_keys=True)
             data_file.write("\n")