Revival: modernize SpiderFoot for Python 3.12+, switch to uv, fix broken modules

- Migrate from requirements.txt to pyproject.toml with relaxed dependency bounds
- Switch to uv for package management across project and Docker
- Update Docker: Alpine 3.20, Python 3.12-bookworm, fix Node/Wappalyzer setup
- Fix secure library API for 1.x (was 0.3.x), PyPDF2 -> pypdf API
- Fix 14 type()==Y patterns to isinstance()
- Cherry-pick upstream bug fixes: WhatsMyName fields (smicallef#1894), nmap parsing (smicallef#1879),
  DNS for Family IP (smicallef#1872), nuclei/wafw00f/whatweb JSON parsing (smicallef#1952),
  db.py UnboundLocalError (smicallef#1787), dev port correlation (smicallef#1827),
  accounts strip_bad_char support (smicallef#1828)
- Add 5 new modules: InternetDB (Shodan free), LeakCheck (paid+free),
  WhoisFreaks, ip2location.io

238/238 modules load successfully on Python 3.12.

Co-Authored-By: Claude Opus 4.6 <[email protected]>

Author: cbxss

Dockerfile

@@ -34,22 +34,20 @@
 #   sudo docker build -t spiderfoot-test --build-arg REQUIREMENTS=test/requirements.txt .
 #   sudo docker run --rm spiderfoot-test -m pytest --flake8 .
 
-FROM alpine:3.12.4 AS build
+FROM alpine:3.20 AS build
 ARG REQUIREMENTS=requirements.txt
 RUN apk add --no-cache gcc git curl python3 python3-dev py3-pip swig tinyxml-dev \
- python3-dev musl-dev openssl-dev libffi-dev libxslt-dev libxml2-dev jpeg-dev \
+ musl-dev openssl-dev libffi-dev libxslt-dev libxml2-dev jpeg-dev \
  openjpeg-dev zlib-dev cargo rust
-RUN python3 -m venv /opt/venv
-ENV PATH="/opt/venv/bin":$PATH
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
+RUN uv venv /opt/venv
+ENV PATH="/opt/venv/bin:$PATH"
+ENV VIRTUAL_ENV="/opt/venv"
 COPY $REQUIREMENTS requirements.txt ./
-RUN ls
-RUN echo "$REQUIREMENTS"
-RUN pip3 install -U pip
-RUN pip3 install -r "$REQUIREMENTS"
-
+RUN uv pip install -r "$REQUIREMENTS"
 
 
-FROM alpine:3.13.0
+FROM alpine:3.20
 WORKDIR /home/spiderfoot
 
 # Place database and logs outside installation directory

Dockerfile.full

@@ -6,7 +6,7 @@
 # Written by: TheTechromancer
 #
 
-FROM python:3
+FROM python:3.12-bookworm
 
 # Install tools/dependencies from apt
 RUN apt-get -y update && apt-get -y install nbtscan onesixtyone nmap
@@ -32,15 +32,8 @@ RUN groupadd spiderfoot \
     && useradd -m -g spiderfoot -d /home/spiderfoot -s /sbin/nologin \
     -c "SpiderFoot User" spiderfoot
 
-# Install RetireJS
-RUN apt remove -y cmdtest \
-    && apt remove -y yarn \
-    && curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \
-    && echo 'deb https://dl.yarnpkg.com/debian/ stable main' |tee /etc/apt/sources.list.d/yarn.list \
-    && apt-get update \
-    && apt-get install yarn -y \
-    && yarn install \
-    && curl -fsSL https://deb.nodesource.com/setup_17.x | bash - \
+# Install Node.js 20 via NodeSource and RetireJS
+RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \
     && apt-get install -y nodejs \
     && npm install -g retire
 
@@ -49,22 +42,23 @@ RUN wget -qO - https://dl.google.com/linux/linux_signing_key.pub | gpg --dearmor
     && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/googlechrome-linux-keyring.gpg] http://dl.google.com/linux/chrome/deb/ stable main" | tee /etc/apt/sources.list.d/google-chrome.list \
     && apt -y update && apt install --allow-unauthenticated -y google-chrome-stable
 
-# Install Wappalyzer
-RUN git clone https://github.com/AliasIO/wappalyzer.git \
-    && cd wappalyzer \
-    && yarn install && yarn run link
+# Install Wappalyzer CLI
+RUN npm install -g wappalyzer-cli
 
 # Install Nuclei
 RUN wget https://github.com/projectdiscovery/nuclei/releases/download/v2.6.5/nuclei_2.6.5_linux_amd64.zip \
     && unzip nuclei_2.6.5_linux_amd64.zip \
-    && git clone https://github.com/projectdiscovery/nuclei-templates.git
+    && for i in 1 2 3; do git clone https://github.com/projectdiscovery/nuclei-templates.git && break || sleep 5; done
 
 # Install testssl.sh
 RUN apt-get install -y bsdmainutils dnsutils coreutils
-RUN git clone https://github.com/drwetter/testssl.sh.git
+RUN for i in 1 2 3; do git clone https://github.com/drwetter/testssl.sh.git && break || sleep 5; done
+
+# Install uv for Python package management
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
 
 # Install Snallygaster and TruffleHog
-RUN pip3 install snallygaster trufflehog
+RUN uv pip install --system snallygaster trufflehog
 
 # Place database and logs outside installation directory
 ENV SPIDERFOOT_DATA /var/lib/spiderfoot
@@ -84,7 +78,7 @@ COPY . .
 ENV VIRTUAL_ENV=/opt/venv
 RUN mkdir -p "$VIRTUAL_ENV" || true
 ENV PATH="$VIRTUAL_ENV/bin:$PATH"
-RUN python -m venv "$VIRTUAL_ENV"
+RUN uv venv "$VIRTUAL_ENV"
 
 ARG REQUIREMENTS=requirements.txt
 COPY "$REQUIREMENTS" requirements.txt
@@ -95,20 +89,17 @@ RUN chown -R spiderfoot:spiderfoot "/home/spiderfoot"
 
 USER spiderfoot
 
-RUN pip install -U pip
-RUN pip install -r "$REQUIREMENTS"
+RUN uv pip install -r "$REQUIREMENTS"
 
 # Install Python tools
-RUN pip install dnstwist
+RUN uv pip install dnstwist
 # CMSeeK
 WORKDIR /tools
 RUN git clone https://github.com/Tuhinshubhra/CMSeeK && cd CMSeeK \
-    && pip install -r requirements.txt && mkdir Results
+    && uv pip install -r requirements.txt && mkdir Results
 
 # Install wafw00f
-RUN git clone https://github.com/EnableSecurity/wafw00f \
-    && cd wafw00f \
-    && python3 setup.py install
+RUN uv pip install wafw00f
 WORKDIR /home/spiderfoot
 
 EXPOSE 5001
@@ -128,7 +119,7 @@ db.configSet({ \
     "sfp_tool_trufflehog:trufflehog_path": "/usr/local/bin/trufflehog", \
     "sfp_tool_nuclei:nuclei_path": "/tools/nuclei", \
     "sfp_tool_nuclei:template_path": "/tools/nuclei-templates", \
-    "sfp_tool_wappalyzer:wappalyzer_path": "/tools/wappalyzer/src/drivers/npm/cli.js", \
+    "sfp_tool_wappalyzer:wappalyzer_path": "/usr/local/bin/wappalyzer", \
     "sfp_tool_nbtscan:nbtscan_path": "/usr/bin/nbtscan", \
     "sfp_tool_nmap:nmappath": "DISABLED_BECAUSE_NMAP_REQUIRES_ROOT_TO_WORK" \
 })' || true && ./sf.py -l 0.0.0.0:5001

correlations/dev_port.yaml

@@ -0,0 +1,25 @@
+id: dev_port
+version: 1
+meta:
+  name: A common port appearing to be a test or development service was found
+  description: >
+    A host with an open tcp
+    port that is often used for development purposes
+    was found. This may indicate a system that is exposed
+    over the Internet unintentionally, and/or may be less secure
+    than other systems.
+  risk: MEDIUM
+collections:
+  - collect:
+      - method: exact
+        field: type
+        value: TCP_PORT_OPEN
+      - method: regex
+        field: data
+        value:
+          - .*:8000$
+          - .*:8080$
+          - .*:8888$
+aggregation:
+  field: source.data
+headline: "Development or internal service found: {source.data}"

docker-compose-dev.yml

@@ -1,5 +1,3 @@
-version: "3"
-
 services:
   spiderfoot:
     volumes:

docker-compose-full.yml

@@ -1,5 +1,3 @@
-version: "3"
-
 services:
   spiderfoot:
     build:

docker-compose.yml

@@ -1,5 +1,3 @@
-version: "3"
-
 # Basic usage:
 #     $ docker-compose up
 #