Skip to content

Commit 034b383

Browse files
ygalnezriygalnezri
committed
Update certstream
1 parent 731291b commit 034b383

File tree

6 files changed

+303
-8
lines changed

6 files changed

+303
-8
lines changed

‎.env‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,8 +98,11 @@ SEARX_COMMAND=-f
9898
# Please add the Watcher Docker containers local subnet network 10.10.10.0/24 in your host server NO_PROXY env variable.
9999

100100
# CertStream URL
101-
CERT_STREAM_URL=wss://certstream.calidog.io
101+
CERT_STREAM_URL=ws://certstream:8080
102102

103103
# If you have a proxy, please fill these variables
104104
HTTP_PROXY=
105105
HTTPS_PROXY=
106+
107+
# Internal traffic bypass proxy
108+
NO_PROXY=certstream,10.10.10.7,localhost,127.0.0.1,10.10.10.3,10.10.10.5,10.10.10.6

‎Watcher/Watcher/dns_finder/certstream_client.py‎

Lines changed: 232 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,232 @@
1+
# coding=utf-8
2+
"""
3+
CertStream WebSocket Client with Enterprise Proxy Support
4+
5+
This module provides a WebSocket client for connecting to certstream-server-go
6+
with automatic proxy detection and internal network bypass.
7+
8+
Compatible with Docker networking and enterprise proxy configurations.
9+
"""
10+
11+
import os
12+
import json
13+
import time
14+
import logging
15+
import threading
16+
from urllib.parse import urlparse
17+
import websocket
18+
from django.conf import settings
19+
20+
logger = logging.getLogger('watcher.dns_finder')
21+
22+
23+
class CertStreamClient:
24+
"""
25+
WebSocket client for CertStream with proxy support and automatic reconnection.
26+
27+
Features:
28+
- Automatic proxy detection and bypass for internal URLs
29+
- Periodic ping to keep connection alive
30+
- Automatic reconnection on failures
31+
- Thread-safe operation
32+
"""
33+
34+
def __init__(self, url=None, callback=None, ping_interval=30, reconnect_delay=5):
35+
"""
36+
Initialize CertStream client.
37+
38+
:param url: WebSocket URL (default: from settings.CERT_STREAM_URL)
39+
:param callback: Callback function to handle messages
40+
:param ping_interval: Seconds between ping messages (0 to disable)
41+
:param reconnect_delay: Seconds to wait before reconnection attempt
42+
"""
43+
self.url = url or getattr(settings, 'CERT_STREAM_URL', 'ws://certstream:8080')
44+
self.callback = callback
45+
self.ping_interval = ping_interval
46+
self.reconnect_delay = reconnect_delay
47+
self.ws = None
48+
self.should_reconnect = True
49+
self.connection_thread = None
50+
51+
# Configure proxy settings
52+
self._setup_proxy()
53+
54+
def _setup_proxy(self):
55+
"""
56+
Configure proxy settings based on environment and URL.
57+
Internal URLs bypass proxy automatically.
58+
"""
59+
# Check if URL is internal (no proxy needed)
60+
if self.is_internal_url(self.url):
61+
logger.info(f"CertStream URL {self.url} is internal - bypassing proxy")
62+
self.http_proxy = None
63+
self.https_proxy = None
64+
else:
65+
# Use environment proxy settings for external URLs
66+
self.http_proxy = os.environ.get('HTTP_PROXY') or os.environ.get('http_proxy')
67+
self.https_proxy = os.environ.get('HTTPS_PROXY') or os.environ.get('https_proxy')
68+
if self.http_proxy or self.https_proxy:
69+
logger.info(f"Using proxy for external CertStream connection")
70+
71+
def is_internal_url(self, url):
72+
"""
73+
Check if URL is internal (Docker service or local network).
74+
75+
:param url: URL to check
76+
:return: True if internal, False otherwise
77+
"""
78+
parsed = urlparse(url)
79+
hostname = parsed.hostname or parsed.netloc.split(':')[0]
80+
81+
# Check NO_PROXY environment variable
82+
no_proxy = os.environ.get('NO_PROXY', '') or os.environ.get('no_proxy', '')
83+
no_proxy_list = [h.strip() for h in no_proxy.split(',') if h.strip()]
84+
85+
# Check if hostname matches NO_PROXY entries
86+
for no_proxy_host in no_proxy_list:
87+
if hostname == no_proxy_host:
88+
return True
89+
# Check for domain suffix match (e.g., .docker.internal)
90+
if no_proxy_host.startswith('.') and hostname.endswith(no_proxy_host):
91+
return True
92+
93+
# Check for common internal patterns
94+
internal_patterns = [
95+
'localhost',
96+
'127.',
97+
'10.',
98+
'172.16.',
99+
'172.17.',
100+
'172.18.',
101+
'172.19.',
102+
'172.20.',
103+
'172.21.',
104+
'172.22.',
105+
'172.23.',
106+
'172.24.',
107+
'172.25.',
108+
'172.26.',
109+
'172.27.',
110+
'172.28.',
111+
'172.29.',
112+
'172.30.',
113+
'172.31.',
114+
'192.168.',
115+
'certstream', # Docker service name
116+
]
117+
118+
for pattern in internal_patterns:
119+
if hostname.startswith(pattern):
120+
return True
121+
122+
return False
123+
124+
def _on_message(self, ws, message):
125+
"""Handle incoming WebSocket messages."""
126+
try:
127+
data = json.loads(message)
128+
if self.callback and data.get('message_type') == 'certificate_update':
129+
self.callback(data, None)
130+
except json.JSONDecodeError as e:
131+
logger.error(f"Failed to decode CertStream message: {e}")
132+
except Exception as e:
133+
logger.error(f"Error in CertStream callback: {e}")
134+
135+
def _on_error(self, ws, error):
136+
"""Handle WebSocket errors."""
137+
logger.error(f"CertStream WebSocket error: {error}")
138+
139+
def _on_close(self, ws, close_status_code, close_msg):
140+
"""Handle WebSocket connection close."""
141+
logger.warning(f"CertStream connection closed: {close_status_code} - {close_msg}")
142+
if self.should_reconnect:
143+
logger.info(f"Reconnecting in {self.reconnect_delay} seconds...")
144+
time.sleep(self.reconnect_delay)
145+
self._connect()
146+
147+
def _on_open(self, ws):
148+
"""Handle WebSocket connection open."""
149+
logger.info(f"CertStream connection established to {self.url}")
150+
151+
# Start ping thread if enabled
152+
if self.ping_interval > 0:
153+
def ping_loop():
154+
while self.ws and self.ws.sock and self.ws.sock.connected:
155+
try:
156+
self.ws.ping()
157+
time.sleep(self.ping_interval)
158+
except Exception as e:
159+
logger.debug(f"Ping failed: {e}")
160+
break
161+
162+
ping_thread = threading.Thread(target=ping_loop, daemon=True)
163+
ping_thread.start()
164+
165+
def _connect(self):
166+
"""Establish WebSocket connection with proxy support."""
167+
try:
168+
# Prepare proxy configuration
169+
proxy_kwargs = {}
170+
if not self.is_internal_url(self.url):
171+
if self.http_proxy:
172+
proxy_kwargs['http_proxy_host'] = urlparse(self.http_proxy).hostname
173+
proxy_kwargs['http_proxy_port'] = urlparse(self.http_proxy).port or 8080
174+
175+
# Create WebSocket connection
176+
self.ws = websocket.WebSocketApp(
177+
self.url,
178+
on_message=self._on_message,
179+
on_error=self._on_error,
180+
on_close=self._on_close,
181+
on_open=self._on_open
182+
)
183+
184+
# Run WebSocket connection (blocking)
185+
self.ws.run_forever(**proxy_kwargs)
186+
187+
except Exception as e:
188+
logger.error(f"Failed to connect to CertStream: {e}")
189+
if self.should_reconnect:
190+
time.sleep(self.reconnect_delay)
191+
self._connect()
192+
193+
def start(self):
194+
"""Start CertStream client in background thread."""
195+
if self.connection_thread and self.connection_thread.is_alive():
196+
logger.warning("CertStream client already running")
197+
return
198+
199+
self.should_reconnect = True
200+
self.connection_thread = threading.Thread(target=self._connect, daemon=True)
201+
self.connection_thread.start()
202+
logger.info("CertStream client started in background")
203+
204+
def stop(self):
205+
"""Stop CertStream client."""
206+
self.should_reconnect = False
207+
if self.ws:
208+
self.ws.close()
209+
logger.info("CertStream client stopped")
210+
211+
212+
def listen_for_events(callback, url=None):
213+
"""
214+
Listen for CertStream events (blocking function).
215+
216+
This is a compatibility function that matches the certstream library API.
217+
218+
:param callback: Function to call for each certificate event
219+
:param url: WebSocket URL (default: from settings.CERT_STREAM_URL)
220+
"""
221+
client = CertStreamClient(url=url, callback=callback)
222+
223+
# Configure NO_PROXY environment to ensure internal connections work
224+
no_proxy = os.environ.get('NO_PROXY', '')
225+
if 'certstream' not in no_proxy:
226+
os.environ['NO_PROXY'] = f"{no_proxy},certstream,10.10.10.7" if no_proxy else "certstream,10.10.10.7"
227+
logger.info(f"Updated NO_PROXY: {os.environ['NO_PROXY']}")
228+
229+
logger.info(f"Starting CertStream listener on {client.url}")
230+
231+
# Start client (blocking call)
232+
client._connect()

‎Watcher/Watcher/dns_finder/core.py‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
import tzlocal
1111
from .models import Alert, DnsMonitored, DnsTwisted, Subscriber, KeywordMonitored
1212
from common.models import LegitimateDomain
13-
import certstream
13+
from . import certstream_client
1414
from common.core import send_app_specific_notifications
1515
from common.core import send_app_specific_notifications_group
1616
from common.core import send_only_thehive_notifications
@@ -116,9 +116,23 @@ def print_callback(message, context):
116116

117117
def main_certificate_transparency():
118118
"""
119-
Launch CertStream scan.
119+
Launch CertStream scan using internal certstream-server-go.
120+
121+
Connects to the internal WebSocket server with automatic proxy bypass.
120122
"""
121-
certstream.listen_for_events(print_callback, url=settings.CERT_STREAM_URL)
123+
# Ensure NO_PROXY is set for internal connections
124+
no_proxy = os.environ.get('NO_PROXY', '')
125+
if 'certstream' not in no_proxy:
126+
os.environ['NO_PROXY'] = f"{no_proxy},certstream,10.10.10.7" if no_proxy else "certstream,10.10.10.7"
127+
128+
logger.info(f"Connecting to certstream-server-go at {settings.CERT_STREAM_URL}")
129+
logger.info(f"NO_PROXY configured: {os.environ.get('NO_PROXY', 'Not set')}")
130+
131+
try:
132+
certstream_client.listen_for_events(print_callback, url=settings.CERT_STREAM_URL)
133+
except Exception as e:
134+
logger.error(f"CertStream connection failed: {e}")
135+
raise
122136

123137

124138
def main_dns_twist():

‎Watcher/requirements.txt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,8 @@ python-tlsh==4.5.0
2525
# Networking and DNS Utilities
2626
dnspython==2.8.0
2727
dnstwist==20250130
28-
certstream==1.12
28+
# certstream==1.12 # Replaced by internal certstream-server-go + websocket-client
29+
websocket-client==1.8.0
2930
tldextract==5.3.0
3031

3132
# Environment Configuration

‎certstream-config.yaml‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
# Configuration for certstream-server-go
2+
# This file is mounted read-only in the Docker container
3+
4+
webserver:
5+
# Listen on all interfaces for Docker networking
6+
listen_addr: "0.0.0.0"
7+
listen_port: 8080
8+
9+
# WebSocket endpoints
10+
lite_url: "/" # Recommended endpoint for Watcher
11+
full_url: "/full-stream" # Full stream with all certificate data
12+
domains_only_url: "/domains-only" # Only domain names
13+
14+
# Enable real IP detection for reverse proxy scenarios
15+
real_ip: true
16+
17+
# Prometheus metrics for monitoring
18+
prometheus:
19+
enabled: true
20+
listen_addr: "0.0.0.0"
21+
listen_port: 9090
22+
metrics_url: "/metrics"

‎docker-compose.yml‎

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,24 @@
11
services:
22

3+
certstream:
4+
container_name: certstream
5+
image: 0rickyy0/certstream-server-go:latest
6+
restart: always
7+
networks:
8+
default:
9+
ipv4_address: 10.10.10.7
10+
volumes:
11+
- ./certstream-config.yaml:/app/config.yaml:ro
12+
ports:
13+
- "8080:8080"
14+
healthcheck:
15+
test: ["CMD", "curl", "-f", "http://localhost:8080/"]
16+
interval: 10s
17+
retries: 5
18+
start_period: 10s
19+
environment:
20+
- TZ=${TZ}
21+
322
searx:
423
container_name: searx
524
image: searx/searx:1.1.0-69-75b859d2
@@ -57,8 +76,12 @@ services:
5776
# image: felix83000/watcher:latest
5877

5978
depends_on:
60-
- db_watcher
61-
- searx
79+
certstream:
80+
condition: service_healthy
81+
db_watcher:
82+
condition: service_started
83+
searx:
84+
condition: service_started
6285
restart: always
6386
networks:
6487
default:
@@ -67,7 +90,7 @@ services:
6790
env_file:
6891
- .env
6992
environment:
70-
no_proxy: "10.10.10.3,10.10.10.5,10.10.10.7"
93+
no_proxy: "10.10.10.3,10.10.10.5,10.10.10.6,10.10.10.7,certstream"
7194
ports:
7295
- "9002:9002"
7396
command: sh -c '/tmp/wait-for-mysql.sh db_watcher 3306 ${DB_USER} ${DB_PASSWORD} db_watcher -- python manage.py runserver 0.0.0.0:9002'

0 commit comments

Comments
 (0)