Skip to content

Commit d4af5d8

Browse files
oleglazarioleglazari
authored
Merge pull request #36 from VRIG-RITSEC/distributed-database
database (just database)
2 parents ffe557b + 0127400 commit d4af5d8

34 files changed

+2885
-2521
lines changed

‎.env‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
# Distributed Fuzzing Environment Configuration
2+
# Copy this file to .env and modify as needed
3+
4+
# Master PostgreSQL Database Configuration
5+
POSTGRES_PASSWORD=fuzzilli123
6+
7+
# Fuzzer Configuration
8+
FUZZER_COUNT=3
9+
SYNC_INTERVAL=300
10+
TIMEOUT=2500
11+
MIN_MUTATIONS_PER_SAMPLE=25
12+
13+
# Optional: Override default fuzzer instance names
14+
# FUZZER_INSTANCE_NAMES=fuzzer-1,fuzzer-2,fuzzer-3
15+
16+
# Optional: Custom V8 revision
17+
# V8_REVISION=b0157a634e584163cbe6004db3161dc16dea20f9
18+
19+
# Optional: Resource limits
20+
# FUZZER_MEMORY_LIMIT=2G
21+
# FUZZER_MEMORY_RESERVATION=1G
22+
23+
# Optional: Enable debug logging
24+
# DEBUG_LOGGING=true

‎Cloud/VRIG/Dockerfile.distributed‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
# Stage 1: Build Fuzzilli
2+
FROM docker.io/swift:latest
3+
4+
ENV DEBIAN_FRONTEND=noninteractive
5+
ENV SHELL=bash
6+
7+
RUN apt-get -y update && apt-get -y upgrade
8+
RUN apt-get -y install nodejs
9+
10+
RUN useradd -m builder
11+
WORKDIR /home/builder
12+
13+
ADD .. fuzzillai
14+
15+
# build Fuzzilli
16+
RUN cd fuzzillai && \
17+
# Temporarily remove test target to avoid build issues
18+
sed -i '/\.testTarget(name: "FuzzilliTests"/,/),$/d' Package.swift && \
19+
swift build -c release --product FuzzilliCli && \
20+
# Verify the executable was created
21+
ls -la .build/release/ && \
22+
find .build -name "FuzzilliCli" -type f -executable
23+
24+
#####################
25+
# Stage 2: Runtime
26+
FROM docker.io/swift:latest
27+
28+
ENV DEBIAN_FRONTEND=noninteractive
29+
ENV SHELL=bash
30+
31+
RUN apt-get -y update && apt-get -y upgrade
32+
33+
RUN useradd -m app
34+
35+
WORKDIR /home/app
36+
37+
# Copy Fuzzilli executable
38+
COPY --from=0 /home/builder/fuzzillai/.build/release/FuzzilliCli FuzzilliCli
39+
40+
# Create directory for V8 build (will be mounted from host)
41+
RUN mkdir -p ./fuzzbuild
42+
43+
RUN mkdir -p ./Corpus
44+
45+
# Environment variables for distributed fuzzing
46+
ENV POSTGRES_URL=postgresql://fuzzilli:fuzzilli123@postgres-master:5432/fuzzilli_master
47+
ENV FUZZER_INSTANCE_NAME=fuzzer-default
48+
ENV TIMEOUT=2500
49+
ENV MIN_MUTATIONS_PER_SAMPLE=25
50+
ENV DEBUG_LOGGING=false
51+
52+
# Default command for distributed fuzzing
53+
# Single master database mode - all workers connect directly to master
54+
CMD ./FuzzilliCli --profile=v8debug --engine=multi --resume --corpus=postgresql \
55+
--postgres-url="${POSTGRES_URL}" \
56+
--timeout="${TIMEOUT}" \
57+
--minMutationsPerSample="${MIN_MUTATIONS_PER_SAMPLE}" \
58+
--postgres-logging ./fuzzbuild/d8

‎Scripts/RunFuzzilli.sh‎

Lines changed: 0 additions & 1 deletion
This file was deleted.

‎Scripts/SetupPostgres.sh‎

Lines changed: 0 additions & 103 deletions
This file was deleted.

‎Scripts/benchmark-server.sh‎

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
#!/bin/bash
2+
3+
# benchmark-server.sh - Benchmark server to determine optimal worker count
4+
# Usage: ./Scripts/benchmark-server.sh
5+
6+
set -e
7+
8+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
9+
PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
10+
11+
# Colors
12+
RED='\033[0;31m'
13+
GREEN='\033[0;32m'
14+
YELLOW='\033[1;33m'
15+
BLUE='\033[0;34m'
16+
CYAN='\033[0;36m'
17+
NC='\033[0m'
18+
19+
echo -e "${CYAN}========================================${NC}"
20+
echo -e "${CYAN} Server Performance Benchmark${NC}"
21+
echo -e "${CYAN}========================================${NC}"
22+
echo ""
23+
24+
# Get system specs
25+
echo -e "${BLUE}=== System Specifications ===${NC}"
26+
CPU_CORES=$(nproc)
27+
TOTAL_MEM=$(free -h | grep Mem | awk '{print $2}')
28+
AVAIL_MEM=$(free -h | grep Mem | awk '{print $7}')
29+
DISK_SPACE=$(df -h / | tail -1 | awk '{print $4}')
30+
31+
echo -e "CPU Cores: ${GREEN}${CPU_CORES}${NC}"
32+
echo -e "Total Memory: ${GREEN}${TOTAL_MEM}${NC}"
33+
echo -e "Available Memory: ${GREEN}${AVAIL_MEM}${NC}"
34+
echo -e "Disk Space: ${GREEN}${DISK_SPACE}${NC}"
35+
echo ""
36+
37+
# Estimate resource usage per worker
38+
echo -e "${BLUE}=== Resource Estimation ===${NC}"
39+
echo "Estimating resource usage per fuzzer worker..."
40+
echo ""
41+
42+
# Typical resource usage per worker (can be adjusted based on observations)
43+
ESTIMATED_CPU_PER_WORKER=15 # percentage
44+
ESTIMATED_MEM_PER_WORKER=512 # MB
45+
ESTIMATED_DB_CONNECTIONS_PER_WORKER=5
46+
47+
# Calculate recommendations
48+
MAX_WORKERS_BY_CPU=$((CPU_CORES * 100 / ESTIMATED_CPU_PER_WORKER))
49+
AVAIL_MEM_MB=$(free -m | grep Mem | awk '{print $7}')
50+
MAX_WORKERS_BY_MEM=$((AVAIL_MEM_MB / ESTIMATED_MEM_PER_WORKER))
51+
MAX_DB_CONNECTIONS=100 # PostgreSQL default max_connections
52+
MAX_WORKERS_BY_DB=$((MAX_DB_CONNECTIONS / ESTIMATED_DB_CONNECTIONS_PER_WORKER))
53+
54+
# Conservative estimate (use minimum)
55+
RECOMMENDED_WORKERS=$((MAX_WORKERS_BY_CPU < MAX_WORKERS_BY_MEM ? MAX_WORKERS_BY_CPU : MAX_WORKERS_BY_MEM))
56+
RECOMMENDED_WORKERS=$((RECOMMENDED_WORKERS < MAX_WORKERS_BY_DB ? RECOMMENDED_WORKERS : MAX_WORKERS_BY_DB))
57+
58+
# Apply safety margin (80% of calculated)
59+
RECOMMENDED_WORKERS=$((RECOMMENDED_WORKERS * 80 / 100))
60+
RECOMMENDED_WORKERS=$((RECOMMENDED_WORKERS > 1 ? RECOMMENDED_WORKERS : 1))
61+
62+
echo -e "Estimated CPU per worker: ${YELLOW}${ESTIMATED_CPU_PER_WORKER}%${NC}"
63+
echo -e "Estimated Memory per worker: ${YELLOW}${ESTIMATED_MEM_PER_WORKER}MB${NC}"
64+
echo -e "Estimated DB conns per worker: ${YELLOW}${ESTIMATED_DB_CONNECTIONS_PER_WORKER}${NC}"
65+
echo ""
66+
67+
echo -e "${BLUE}=== Capacity Analysis ===${NC}"
68+
echo -e "Max workers (CPU): ${CYAN}${MAX_WORKERS_BY_CPU}${NC}"
69+
echo -e "Max workers (Memory): ${CYAN}${MAX_WORKERS_BY_MEM}${NC}"
70+
echo -e "Max workers (DB): ${CYAN}${MAX_WORKERS_BY_DB}${NC}"
71+
echo ""
72+
73+
echo -e "${GREEN}=== Recommended Configuration ===${NC}"
74+
echo -e "Recommended Workers: ${GREEN}${RECOMMENDED_WORKERS}${NC}"
75+
echo ""
76+
77+
# Performance test with current workers
78+
if docker ps --format "{{.Names}}" | grep -q "fuzzer-worker"; then
79+
echo -e "${BLUE}=== Current Performance Test ===${NC}"
80+
echo "Testing current worker performance..."
81+
echo ""
82+
83+
# Get current worker count
84+
CURRENT_WORKERS=$(docker ps --format "{{.Names}}" | grep -c "fuzzer-worker" || echo "0")
85+
echo -e "Current Workers: ${CYAN}${CURRENT_WORKERS}${NC}"
86+
87+
# Monitor for 30 seconds
88+
echo "Monitoring for 30 seconds..."
89+
START_TIME=$(date +%s)
90+
91+
# Get initial stats
92+
if docker ps --format "{{.Names}}" | grep -q "fuzzilli-postgres-master"; then
93+
DB_CONTAINER="fuzzilli-postgres-master"
94+
DB_NAME="fuzzilli_master"
95+
DB_USER="fuzzilli"
96+
97+
INITIAL_EXECS=$(docker exec "$DB_CONTAINER" psql -U "$DB_USER" -d "$DB_NAME" -t -A -c "
98+
SELECT COUNT(*) FROM execution WHERE created_at > NOW() - INTERVAL '1 minute';
99+
" 2>/dev/null || echo "0")
100+
101+
sleep 30
102+
103+
FINAL_EXECS=$(docker exec "$DB_CONTAINER" psql -U "$DB_USER" -d "$DB_NAME" -t -A -c "
104+
SELECT COUNT(*) FROM execution WHERE created_at > NOW() - INTERVAL '1 minute';
105+
" 2>/dev/null || echo "0")
106+
107+
EXECS_PER_SEC=$(( (FINAL_EXECS - INITIAL_EXECS) / 30 ))
108+
EXECS_PER_WORKER=$(( EXECS_PER_SEC / CURRENT_WORKERS ))
109+
110+
echo -e "Executions/sec: ${GREEN}${EXECS_PER_SEC}${NC}"
111+
echo -e "Executions/worker: ${GREEN}${EXECS_PER_WORKER}${NC}"
112+
echo ""
113+
114+
# Get system load
115+
CPU_USAGE=$(top -bn1 | grep "Cpu(s)" | sed "s/.*, *\([0-9.]*\)%* id.*/\1/" | awk '{print 100 - $1}')
116+
MEM_USAGE=$(free | grep Mem | awk '{printf "%.1f", ($3/$2) * 100.0}')
117+
118+
echo -e "CPU Usage: ${CYAN}${CPU_USAGE}%${NC}"
119+
echo -e "Memory Usage: ${CYAN}${MEM_USAGE}%${NC}"
120+
echo ""
121+
122+
# Scaling recommendations
123+
echo -e "${BLUE}=== Scaling Recommendations ===${NC}"
124+
if (( $(echo "$CPU_USAGE < 50" | bc -l) )) && (( $(echo "$MEM_USAGE < 50" | bc -l) )); then
125+
SUGGESTED_WORKERS=$((CURRENT_WORKERS * 2))
126+
echo -e "${GREEN}✓ System has capacity${NC}"
127+
echo -e "Suggested workers: ${GREEN}${SUGGESTED_WORKERS}${NC} (double current)"
128+
elif (( $(echo "$CPU_USAGE > 80" | bc -l) )) || (( $(echo "$MEM_USAGE > 80" | bc -l) )); then
129+
SUGGESTED_WORKERS=$((CURRENT_WORKERS / 2))
130+
SUGGESTED_WORKERS=$((SUGGESTED_WORKERS > 1 ? SUGGESTED_WORKERS : 1))
131+
echo -e "${RED}⚠ System under high load${NC}"
132+
echo -e "Suggested workers: ${YELLOW}${SUGGESTED_WORKERS}${NC} (reduce by half)"
133+
else
134+
echo -e "${YELLOW}System load is moderate${NC}"
135+
echo -e "Current worker count seems appropriate"
136+
fi
137+
fi
138+
else
139+
echo -e "${YELLOW}No workers currently running${NC}"
140+
echo -e "Start workers with: ${CYAN}./Scripts/start-distributed.sh ${RECOMMENDED_WORKERS}${NC}"
141+
fi
142+
143+
echo ""
144+
echo -e "${CYAN}========================================${NC}"
145+

0 commit comments

Comments
 (0)