Skip to content

Commit 8f58cd4

Browse files
oleglazarioleglazari
authored
Merge pull request #38 from VRIG-RITSEC/distributed-database
added fixes for clearing db
2 parents 216056f + 9f23bf0 commit 8f58cd4

File tree

3 files changed

+246
-129
lines changed

3 files changed

+246
-129
lines changed

‎Scripts/clear-db.sh‎

Lines changed: 223 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,223 @@
1+
#!/bin/bash
2+
3+
# Clear PostgreSQL Database Script
4+
# This script clears all data from the Fuzzilli PostgreSQL database
5+
# while preserving the schema and lookup table seed data.
6+
7+
# Don't exit on error - we want to continue even if some operations fail
8+
set +e
9+
10+
# Database connection parameters
11+
DB_CONTAINER="fuzzilli-postgres-master"
12+
DB_NAME="fuzzilli_master"
13+
DB_USER="fuzzilli"
14+
DB_PASSWORD="fuzzilli123"
15+
16+
# Colors for output
17+
RED='\033[0;31m'
18+
GREEN='\033[0;32m'
19+
YELLOW='\033[1;33m'
20+
CYAN='\033[0;36m'
21+
NC='\033[0m' # No Color
22+
23+
# Function to check if Docker is available
24+
check_docker() {
25+
if ! command -v docker &> /dev/null; then
26+
echo -e "${RED}Error: Docker command not found. Please install Docker.${NC}"
27+
exit 1
28+
fi
29+
}
30+
31+
# Function to check if PostgreSQL container is running
32+
check_container() {
33+
if ! docker ps --format "table {{.Names}}" | grep -q "$DB_CONTAINER"; then
34+
echo -e "${RED}Error: PostgreSQL container '$DB_CONTAINER' is not running${NC}"
35+
echo "Available containers:"
36+
docker ps --format "table {{.Names}}\t{{.Status}}"
37+
exit 1
38+
fi
39+
}
40+
41+
# Function to run a query
42+
run_query() {
43+
local query="$1"
44+
docker exec -i "$DB_CONTAINER" psql -U "$DB_USER" -d "$DB_NAME" -c "$query" 2>&1
45+
}
46+
47+
# Function to run a query silently (for operations that might fail)
48+
run_query_silent() {
49+
local query="$1"
50+
docker exec -i "$DB_CONTAINER" psql -U "$DB_USER" -d "$DB_NAME" -c "$query" 2>/dev/null || true
51+
}
52+
53+
# Function to get table row counts
54+
get_table_counts() {
55+
echo -e "${CYAN}Current database statistics:${NC}"
56+
run_query "
57+
SELECT
58+
'main' as table_name, COUNT(*) as row_count FROM main
59+
UNION ALL
60+
SELECT 'fuzzer', COUNT(*) FROM fuzzer
61+
UNION ALL
62+
SELECT 'program', COUNT(*) FROM program
63+
UNION ALL
64+
SELECT 'execution', COUNT(*) FROM execution
65+
UNION ALL
66+
SELECT 'coverage_detail', COUNT(*) FROM coverage_detail
67+
UNION ALL
68+
SELECT 'feedback_vector_detail', COUNT(*) FROM feedback_vector_detail
69+
UNION ALL
70+
SELECT 'crash_analysis', COUNT(*) FROM crash_analysis
71+
UNION ALL
72+
SELECT 'fuzzer_statistics', COALESCE((SELECT COUNT(*) FROM fuzzer_statistics), 0);
73+
" | grep -v "row_count" | grep -v "^$" | grep -v "^-" | while read -r line; do
74+
if [ -n "$line" ]; then
75+
echo " $line"
76+
fi
77+
done
78+
}
79+
80+
# Main execution
81+
main() {
82+
echo -e "${CYAN}========================================${NC}"
83+
echo -e "${CYAN} Fuzzilli Database Cleanup${NC}"
84+
echo -e "${CYAN}========================================${NC}"
85+
echo ""
86+
87+
check_docker
88+
check_container
89+
90+
# Show current statistics
91+
get_table_counts
92+
echo ""
93+
94+
# Find all fuzzer-X tables
95+
fuzzer_tables=$(run_query "SELECT tablename FROM pg_tables WHERE schemaname = 'public' AND tablename LIKE 'fuzzer-%' ORDER BY tablename;" | grep -v "tablename" | grep -v "^-" | grep -v "^$" | grep -v "^(" | awk 'NF>0 {print $1}' | tr '\n' ' ')
96+
97+
# Safety confirmation
98+
if [ "${1:-}" != "--yes" ] && [ "${1:-}" != "-y" ]; then
99+
echo -e "${YELLOW}WARNING: This will delete ALL data from the database!${NC}"
100+
echo -e "${YELLOW}The following will be cleared:${NC}"
101+
echo " - All fuzzer instances (main table)"
102+
echo " - All corpus programs (fuzzer table)"
103+
echo " - All executed programs (program table)"
104+
echo " - All execution records (execution table)"
105+
echo " - All coverage details (coverage_detail table)"
106+
echo " - All feedback vector details (feedback_vector_detail table)"
107+
echo " - All crash analyses (crash_analysis table)"
108+
echo " - All fuzzer statistics (fuzzer_statistics table)"
109+
if [ -n "$fuzzer_tables" ]; then
110+
echo -e "${YELLOW} - All fuzzer-X tables (will be DROPPED):${NC}"
111+
for table in $fuzzer_tables; do
112+
echo " - $table"
113+
done
114+
fi
115+
echo ""
116+
echo -e "${YELLOW}The following will be preserved:${NC}"
117+
echo " - Database schema (tables, indexes, views)"
118+
echo " - Lookup tables (execution_type, mutator_type, execution_outcome)"
119+
echo ""
120+
read -p "Are you sure you want to continue? (yes/no): " confirm
121+
if [ "$confirm" != "yes" ]; then
122+
echo -e "${GREEN}Operation cancelled.${NC}"
123+
exit 0
124+
fi
125+
fi
126+
127+
echo -e "${YELLOW}Clearing database...${NC}"
128+
echo ""
129+
130+
# Clear data tables in order (respecting foreign key constraints)
131+
# Start with child tables first, then parent tables
132+
# Use CASCADE to handle foreign key dependencies
133+
134+
echo -e "${CYAN}Clearing coverage_detail...${NC}"
135+
run_query_silent "TRUNCATE TABLE coverage_detail CASCADE;"
136+
137+
echo -e "${CYAN}Clearing feedback_vector_detail...${NC}"
138+
run_query_silent "TRUNCATE TABLE feedback_vector_detail CASCADE;"
139+
140+
echo -e "${CYAN}Clearing crash_analysis...${NC}"
141+
run_query_silent "TRUNCATE TABLE crash_analysis CASCADE;"
142+
143+
echo -e "${CYAN}Clearing execution...${NC}"
144+
run_query_silent "TRUNCATE TABLE execution CASCADE;"
145+
146+
echo -e "${CYAN}Clearing program...${NC}"
147+
run_query_silent "TRUNCATE TABLE program CASCADE;"
148+
149+
echo -e "${CYAN}Clearing fuzzer (corpus)...${NC}"
150+
run_query_silent "TRUNCATE TABLE fuzzer CASCADE;"
151+
152+
echo -e "${CYAN}Clearing fuzzer_statistics...${NC}"
153+
run_query_silent "TRUNCATE TABLE fuzzer_statistics CASCADE;"
154+
155+
echo -e "${CYAN}Clearing main (fuzzer instances)...${NC}"
156+
run_query_silent "TRUNCATE TABLE main CASCADE;"
157+
158+
# Drop all fuzzer-X tables if they exist
159+
if [ -n "$fuzzer_tables" ]; then
160+
echo -e "${CYAN}Dropping fuzzer-X tables...${NC}"
161+
for table in $fuzzer_tables; do
162+
if [ -n "$table" ]; then
163+
echo " Dropping table: $table"
164+
run_query_silent "DROP TABLE IF EXISTS \"$table\" CASCADE;"
165+
fi
166+
done
167+
else
168+
echo -e "${CYAN}No fuzzer-X tables found to drop.${NC}"
169+
fi
170+
171+
# Reset sequences
172+
echo -e "${CYAN}Resetting sequences...${NC}"
173+
run_query_silent "ALTER SEQUENCE IF EXISTS main_fuzzer_id_seq RESTART WITH 1;"
174+
run_query_silent "ALTER SEQUENCE IF EXISTS execution_execution_id_seq RESTART WITH 1;"
175+
run_query_silent "ALTER SEQUENCE IF EXISTS feedback_vector_detail_id_seq RESTART WITH 1;"
176+
run_query_silent "ALTER SEQUENCE IF EXISTS coverage_detail_id_seq RESTART WITH 1;"
177+
run_query_silent "ALTER SEQUENCE IF EXISTS crash_analysis_id_seq RESTART WITH 1;"
178+
179+
echo ""
180+
echo -e "${GREEN}Database cleared successfully!${NC}"
181+
echo ""
182+
183+
# Show final statistics
184+
get_table_counts
185+
echo ""
186+
187+
echo -e "${CYAN}========================================${NC}"
188+
echo -e "${CYAN} Cleanup Complete${NC}"
189+
echo -e "${CYAN}========================================${NC}"
190+
}
191+
192+
# Handle command line arguments
193+
case "${1:-}" in
194+
"help"|"-h"|"--help")
195+
echo "Usage: $0 [--yes|-y]"
196+
echo ""
197+
echo "Clears all data from the Fuzzilli PostgreSQL database."
198+
echo ""
199+
echo "Options:"
200+
echo " --yes, -y Skip confirmation prompt"
201+
echo " help, -h Show this help message"
202+
echo ""
203+
echo "This script will:"
204+
echo " - Delete all fuzzer instances, programs, executions, and related data"
205+
echo " - Drop all fuzzer-X tables (fuzzer-1, fuzzer-2, etc.)"
206+
echo " - Preserve the database schema and lookup tables"
207+
echo " - Reset all sequences to start from 1"
208+
echo ""
209+
echo "Note: Update DB_CONTAINER variable in script if your container has a different name"
210+
;;
211+
"")
212+
main
213+
;;
214+
"--yes"|"-y")
215+
main "$1"
216+
;;
217+
*)
218+
echo "Unknown option: $1"
219+
echo "Use '$0 help' for usage information"
220+
exit 1
221+
;;
222+
esac
223+

‎Sources/Fuzzilli/Corpus/PostgreSQLCorpus.swift‎

Lines changed: 23 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,8 @@ public class PostgreSQLCorpus: ComponentBase, Corpus {
3333
private var fuzzerRegistered = false
3434
private var fuzzerId: Int? // Master database fuzzer ID
3535

36-
/// Batch execution storage
37-
private var pendingExecutions: [(Program, ProgramAspects, DatabaseExecutionPurpose)] = []
36+
/// Batch execution storage - includes execution metadata
37+
private var pendingExecutions: [(Program, ProgramAspects, DatabaseExecutionPurpose, ExecutionData)] = []
3838
private var executionBatchSize: Int // Dynamic batch size
3939
private let executionBatchLock = NSLock()
4040

@@ -178,7 +178,7 @@ public class PostgreSQLCorpus: ComponentBase, Corpus {
178178
guard fuzzerId != nil else { return }
179179

180180
// Commit pending executions
181-
let queuedExecutions = executionBatchLock.withLock {
181+
let queuedExecutions: [(Program, ProgramAspects, DatabaseExecutionPurpose, ExecutionData)] = executionBatchLock.withLock {
182182
let queuedExecutions = pendingExecutions
183183
pendingExecutions.removeAll()
184184
return queuedExecutions
@@ -390,10 +390,10 @@ public class PostgreSQLCorpus: ComponentBase, Corpus {
390390
}
391391

392392
/// Add execution to batch for later processing
393-
private func addToExecutionBatch(_ program: Program, _ aspects: ProgramAspects, executionType: DatabaseExecutionPurpose) {
393+
private func addToExecutionBatch(_ program: Program, _ aspects: ProgramAspects, executionType: DatabaseExecutionPurpose, executionData: ExecutionData) {
394394
// Use atomic operations to avoid blocking locks
395-
let shouldProcessBatch: [(Program, ProgramAspects, DatabaseExecutionPurpose)]? = executionBatchLock.withLock {
396-
pendingExecutions.append((program, aspects, executionType))
395+
let shouldProcessBatch: [(Program, ProgramAspects, DatabaseExecutionPurpose, ExecutionData)]? = executionBatchLock.withLock {
396+
pendingExecutions.append((program, aspects, executionType, executionData))
397397
let currentBatchSize = executionBatchSize
398398
let shouldProcess = pendingExecutions.count >= currentBatchSize
399399
if shouldProcess {
@@ -413,7 +413,7 @@ public class PostgreSQLCorpus: ComponentBase, Corpus {
413413
}
414414

415415
/// Process a batch of executions
416-
private func processExecutionBatch(_ batch: [(Program, ProgramAspects, DatabaseExecutionPurpose)]) async {
416+
private func processExecutionBatch(_ batch: [(Program, ProgramAspects, DatabaseExecutionPurpose, ExecutionData)]) async {
417417
guard let fuzzerId = fuzzerId else {
418418
logger.error("Cannot process execution batch: fuzzer not registered")
419419
return
@@ -428,7 +428,7 @@ public class PostgreSQLCorpus: ComponentBase, Corpus {
428428
var uniquePrograms: [String: (Program, ExecutionMetadata)] = [:]
429429
var executionBatchData: [ExecutionBatchData] = []
430430

431-
for (program, aspects, executionType) in batch {
431+
for (program, aspects, executionType, execData) in batch {
432432
// Filter out test programs with FUZZILLI_CRASH (false positive crashes)
433433
if DatabaseUtils.containsFuzzilliCrash(program: program) {
434434
if enableLogging {
@@ -464,7 +464,11 @@ public class PostgreSQLCorpus: ComponentBase, Corpus {
464464
mutatorType: nil,
465465
outcome: aspects.outcome,
466466
coverage: coveragePct,
467-
coverageEdges: (aspects as? CovEdgeSet).map { Set($0.getEdges().map { Int($0) }) } ?? Set<Int>()
467+
executionTimeMs: Int(execData.execTime * 1000),
468+
coverageEdges: (aspects as? CovEdgeSet).map { Set($0.getEdges().map { Int($0) }) } ?? Set<Int>(),
469+
stdout: execData.stdout,
470+
stderr: execData.stderr,
471+
fuzzout: execData.fuzzout
468472
)
469473
executionBatchData.append(executionData)
470474
}
@@ -491,7 +495,7 @@ public class PostgreSQLCorpus: ComponentBase, Corpus {
491495

492496
/// Flush any pending executions in the batch
493497
private func flushExecutionBatch() {
494-
let batch = executionBatchLock.withLock {
498+
let batch: [(Program, ProgramAspects, DatabaseExecutionPurpose, ExecutionData)] = executionBatchLock.withLock {
495499
let batch = pendingExecutions
496500
pendingExecutions.removeAll()
497501
return batch
@@ -521,7 +525,7 @@ public class PostgreSQLCorpus: ComponentBase, Corpus {
521525
}
522526

523527
// Process any queued executions
524-
let queuedExecutions = executionBatchLock.withLock {
528+
let queuedExecutions: [(Program, ProgramAspects, DatabaseExecutionPurpose, ExecutionData)] = executionBatchLock.withLock {
525529
let queuedExecutions = pendingExecutions
526530
pendingExecutions.removeAll()
527531
return queuedExecutions
@@ -681,53 +685,15 @@ public class PostgreSQLCorpus: ComponentBase, Corpus {
681685
return
682686
}
683687

684-
do {
685-
// Use the registered fuzzer ID
686-
guard let fuzzerId = fuzzerId else {
687-
if enableLogging {
688-
self.logger.info("Cannot store execution: fuzzer not registered")
689-
}
690-
return
691-
}
692-
693-
// DEBUG: Log execution storage attempt
694-
if enableLogging {
695-
self.logger.info("Storing execution: fuzzerId=\(fuzzerId), outcome=\(executionData.outcome), execTime=\(executionData.execTime)")
696-
}
697-
698-
// Derive coverage percentage (0-100) from evaluator if available
699-
let coveragePct: Double = {
700-
if let coverageEvaluator = self.fuzzer.evaluator as? ProgramCoverageEvaluator {
701-
return coverageEvaluator.currentScore * 100.0
702-
} else {
703-
return 0.0
704-
}
705-
}()
706-
707-
// Store both program and execution in a single transaction to avoid foreign key issues
708-
_ = try await storage.storeProgramAndExecution(
709-
program: program,
710-
fuzzerId: fuzzerId,
711-
executionType: executionType,
712-
outcome: executionData.outcome,
713-
coverage: coveragePct,
714-
executionTimeMs: Int(executionData.execTime * 1000),
715-
stdout: executionData.stdout,
716-
stderr: executionData.stderr,
717-
fuzzout: executionData.fuzzout,
718-
metadata: ExecutionMetadata(lastOutcome: DatabaseExecutionOutcome(
719-
id: DatabaseUtils.mapExecutionOutcome(outcome: aspects.outcome),
720-
outcome: aspects.outcome.description,
721-
description: aspects.outcome.description
722-
))
723-
)
724-
725-
if enableLogging {
726-
self.logger.info("Successfully stored program and execution")
688+
// Add to execution batch for batched storage (even if fuzzer not registered yet)
689+
addToExecutionBatch(program, aspects, executionType: executionType, executionData: executionData)
690+
691+
if enableLogging {
692+
if fuzzerId == nil {
693+
self.logger.info("Queued execution for later (fuzzer not registered yet)")
694+
} else {
695+
self.logger.info("Added execution to batch: outcome=\(executionData.outcome), execTime=\(executionData.execTime)")
727696
}
728-
729-
} catch {
730-
logger.error("Failed to store execution: \(String(reflecting: error))")
731697
}
732698
}
733699

0 commit comments

Comments
 (0)