Skip to content

Commit fe96ec4

Browse files
author
dudcom
committed
new proper rag-embeddings
1 parent 16e7eba commit fe96ec4

File tree

7 files changed

+3592
-26
lines changed

7 files changed

+3592
-26
lines changed

‎Scripts/performance-analysis.sh‎

Lines changed: 268 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,268 @@
1+
#!/bin/bash
2+
3+
# performance-analysis.sh - Quick performance analysis tool for Fuzzilli
4+
# Usage: ./Scripts/performance-analysis.sh [command]
5+
#
6+
# Commands:
7+
# monitor - Real-time system and fuzzing metrics
8+
# stats - Database statistics dashboard
9+
# profile - CPU profiling with flamegraph (requires running fuzzer)
10+
# database - Database performance queries
11+
# containers - Container resource usage
12+
# all - Run all analysis tools
13+
14+
set -e
15+
16+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
17+
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
18+
19+
# Colors
20+
RED='\033[0;31m'
21+
GREEN='\033[0;32m'
22+
YELLOW='\033[1;33m'
23+
BLUE='\033[0;34m'
24+
CYAN='\033[0;36m'
25+
NC='\033[0m'
26+
27+
show_usage() {
28+
echo -e "${CYAN}Performance Analysis Tool for Fuzzilli${NC}"
29+
echo ""
30+
echo "Usage: $0 [command]"
31+
echo ""
32+
echo "Commands:"
33+
echo " monitor - Real-time system and fuzzing metrics (interactive)"
34+
echo " stats - Database statistics dashboard (interactive)"
35+
echo " profile - CPU profiling with flamegraph (requires running fuzzer)"
36+
echo " database - Database performance queries"
37+
echo " containers - Container resource usage"
38+
echo " quick - Quick performance snapshot"
39+
echo " all - Run all non-interactive analysis"
40+
echo ""
41+
echo "Examples:"
42+
echo " $0 monitor # Start real-time monitoring"
43+
echo " $0 stats # View statistics dashboard"
44+
echo " $0 profile # Profile CPU and generate flamegraph"
45+
echo " $0 quick # Quick snapshot of current performance"
46+
}
47+
48+
run_monitor() {
49+
echo -e "${GREEN}Starting real-time performance monitor...${NC}"
50+
"${SCRIPT_DIR}/monitor-performance.sh"
51+
}
52+
53+
run_stats() {
54+
echo -e "${GREEN}Starting statistics dashboard...${NC}"
55+
"${SCRIPT_DIR}/fuzzer-stats.sh"
56+
}
57+
58+
run_profile() {
59+
echo -e "${GREEN}Starting CPU profiling...${NC}"
60+
61+
# Check if perf is available
62+
if ! command -v perf &> /dev/null; then
63+
echo -e "${RED}Error: perf tool not found. Install with: sudo apt-get install linux-perf${NC}"
64+
exit 1
65+
fi
66+
67+
# Check if perf can run (perf_event_paranoid check)
68+
if ! perf record --help &> /dev/null; then
69+
echo -e "${YELLOW}Warning: perf may require sudo privileges${NC}"
70+
echo -e "${CYAN}If you see 'perf_event_paranoid' errors, run with:${NC}"
71+
echo " sudo $0 profile"
72+
echo ""
73+
echo -e "${CYAN}Or adjust perf_event_paranoid (temporary):${NC}"
74+
echo " sudo sysctl -w kernel.perf_event_paranoid=-1"
75+
echo ""
76+
fi
77+
78+
# Check if perf_fuzzilli.sh exists
79+
if [ -f "/home/aleksi/perf_fuzzilli.sh" ]; then
80+
# Make sure it's executable, or run with bash
81+
if [ -x "/home/aleksi/perf_fuzzilli.sh" ]; then
82+
/home/aleksi/perf_fuzzilli.sh
83+
else
84+
echo -e "${YELLOW}perf_fuzzilli.sh not executable, running with bash...${NC}"
85+
bash /home/aleksi/perf_fuzzilli.sh
86+
fi
87+
else
88+
echo -e "${YELLOW}perf_fuzzilli.sh not found. Creating profile...${NC}"
89+
90+
# Find FuzzilliCli process
91+
FUZZILLI_PID=$(docker exec fuzzer-worker-1 pgrep -f FuzzilliCli 2>/dev/null | head -n 1)
92+
93+
if [ -z "$FUZZILLI_PID" ]; then
94+
echo -e "${RED}Error: FuzzilliCli not running. Start fuzzers first.${NC}"
95+
exit 1
96+
fi
97+
98+
echo "Profiling PID $FUZZILLI_PID for 8 minutes..."
99+
DURATION=1200
100+
101+
# Note: perf needs to run on host, not in container
102+
echo -e "${YELLOW}Note: perf must run on host system. Use:${NC}"
103+
echo " sudo perf record -F 99 --call-graph dwarf -p <PID> -o perf.data -- sleep $DURATION"
104+
echo " perf script -i perf.data > out.perf"
105+
echo " /home/aleksi/FlameGraph/stackcollapse-perf.pl out.perf | swift demangle > out.folded"
106+
echo " /home/aleksi/FlameGraph/flamegraph.pl --title 'Fuzzilli Performance' out.folded > fuzzilli_flamegraph.svg"
107+
fi
108+
}
109+
110+
run_database() {
111+
echo -e "${GREEN}Database Performance Analysis${NC}"
112+
echo ""
113+
114+
DB_CONTAINER="fuzzilli-postgres-master"
115+
DB_NAME="fuzzilli_master"
116+
DB_USER="fuzzilli"
117+
118+
if ! docker ps --format "{{.Names}}" | grep -q "^${DB_CONTAINER}$"; then
119+
echo -e "${RED}Error: Database container not running${NC}"
120+
exit 1
121+
fi
122+
123+
echo -e "${CYAN}=== Database Size ===${NC}"
124+
docker exec "$DB_CONTAINER" psql -U "$DB_USER" -d "$DB_NAME" -c "
125+
SELECT pg_size_pretty(pg_database_size('$DB_NAME')) as database_size;
126+
"
127+
128+
echo ""
129+
echo -e "${CYAN}=== Active Connections ===${NC}"
130+
docker exec "$DB_CONTAINER" psql -U "$DB_USER" -d "$DB_NAME" -c "
131+
SELECT count(*) as active_connections
132+
FROM pg_stat_activity
133+
WHERE datname = '$DB_NAME';
134+
"
135+
136+
echo ""
137+
echo -e "${CYAN}=== Table Sizes ===${NC}"
138+
docker exec "$DB_CONTAINER" psql -U "$DB_USER" -d "$DB_NAME" -c "
139+
SELECT
140+
tablename,
141+
pg_size_pretty(pg_total_relation_size('public.'||tablename)) AS size
142+
FROM pg_tables
143+
WHERE schemaname = 'public'
144+
ORDER BY pg_total_relation_size('public.'||tablename) DESC
145+
LIMIT 10;
146+
"
147+
148+
echo ""
149+
echo -e "${CYAN}=== Slow Queries (if enabled) ===${NC}"
150+
docker exec "$DB_CONTAINER" psql -U "$DB_USER" -d "$DB_NAME" -c "
151+
SELECT
152+
query,
153+
mean_exec_time,
154+
calls
155+
FROM pg_stat_statements
156+
ORDER BY mean_exec_time DESC
157+
LIMIT 5;
158+
" 2>/dev/null || echo "pg_stat_statements not enabled"
159+
160+
echo ""
161+
echo -e "${CYAN}=== Index Usage ===${NC}"
162+
docker exec "$DB_CONTAINER" psql -U "$DB_USER" -d "$DB_NAME" -c "
163+
SELECT
164+
tablename,
165+
indexname,
166+
idx_scan
167+
FROM pg_stat_user_indexes
168+
WHERE idx_scan = 0
169+
ORDER BY pg_relation_size(indexrelid) DESC
170+
LIMIT 10;
171+
"
172+
}
173+
174+
run_containers() {
175+
echo -e "${GREEN}Container Resource Usage${NC}"
176+
echo ""
177+
178+
echo -e "${CYAN}=== All Containers ===${NC}"
179+
docker stats --no-stream --format "table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.NetIO}}\t{{.BlockIO}}"
180+
181+
echo ""
182+
echo -e "${CYAN}=== Fuzzer Workers Only ===${NC}"
183+
docker stats --no-stream --format "table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}" $(docker ps --format "{{.Names}}" | grep fuzzer-worker)
184+
}
185+
186+
run_quick() {
187+
echo -e "${GREEN}Quick Performance Snapshot${NC}"
188+
echo ""
189+
190+
# System resources
191+
echo -e "${CYAN}=== System Resources ===${NC}"
192+
CPU=$(top -bn1 | grep "Cpu(s)" | sed "s/.*, *\([0-9.]*\)%* id.*/\1/" | awk '{print 100 - $1}')
193+
MEM=$(free | grep Mem | awk '{printf "%.1f", ($3/$2) * 100.0}')
194+
echo "CPU Usage: ${CPU}%"
195+
echo "Memory Usage: ${MEM}%"
196+
echo ""
197+
198+
# Container count
199+
echo -e "${CYAN}=== Containers ===${NC}"
200+
WORKER_COUNT=$(docker ps --format "{{.Names}}" | grep -c fuzzer-worker || echo "0")
201+
echo "Active Workers: $WORKER_COUNT"
202+
echo ""
203+
204+
# Database stats
205+
DB_CONTAINER="fuzzilli-postgres-master"
206+
DB_NAME="fuzzilli_master"
207+
DB_USER="fuzzilli"
208+
209+
if docker ps --format "{{.Names}}" | grep -q "^${DB_CONTAINER}$"; then
210+
echo -e "${CYAN}=== Database ===${NC}"
211+
EXECS_LAST_HOUR=$(docker exec "$DB_CONTAINER" psql -U "$DB_USER" -d "$DB_NAME" -t -A -c "
212+
SELECT COUNT(*) FROM execution WHERE created_at > NOW() - INTERVAL '1 hour';
213+
" 2>/dev/null || echo "0")
214+
EXECS_PER_SEC=$(echo "scale=2; $EXECS_LAST_HOUR / 3600" | bc 2>/dev/null || echo "0")
215+
echo "Executions (last hour): $EXECS_LAST_HOUR"
216+
echo "Executions/sec: $EXECS_PER_SEC"
217+
fi
218+
}
219+
220+
run_all() {
221+
echo -e "${GREEN}Running all performance analysis...${NC}"
222+
echo ""
223+
224+
run_quick
225+
echo ""
226+
run_database
227+
echo ""
228+
run_containers
229+
}
230+
231+
# Main
232+
if [ $# -eq 0 ]; then
233+
show_usage
234+
exit 0
235+
fi
236+
237+
COMMAND=$1
238+
shift
239+
240+
case "$COMMAND" in
241+
monitor)
242+
run_monitor
243+
;;
244+
stats)
245+
run_stats
246+
;;
247+
profile)
248+
run_profile
249+
;;
250+
database)
251+
run_database
252+
;;
253+
containers)
254+
run_containers
255+
;;
256+
quick)
257+
run_quick
258+
;;
259+
all)
260+
run_all
261+
;;
262+
*)
263+
echo -e "${RED}Unknown command: $COMMAND${NC}"
264+
echo ""
265+
show_usage
266+
exit 1
267+
;;
268+
esac

‎Sources/Agentic_System/rag_db/v8_source_rag/v8_source_rag_metadata.json‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
[
2+
{"doc_id": "23ea7b870d74c8c6ec2ed6ae6b79556e771926f4", "path": "regexp/regexp-utils.h", "parent_file": "regexp/regexp-utils.h", "parent_id": "36e4aa48dfb01fe5f5d9f9b0c3c9f35a6cbfa2d5", "topic": "V8 regexp", "doc_type": "code", "source": "v8_source", "content": "// Copyright 2016 the V8 project authors. All rights reserved.\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\n#ifndef V8_REGEXP_REGEXP_UTILS_H_\n#define V8_REGEXP_REGEXP_UTILS_H_\n\n#include \"src/common/globals.h\"\n\nnamespace v8 {\nnamespace internal {\n\nclass JSReceiver;\nclass Object;\nclass RegExpMatchInfo;\nclass String;\n\n// Helper methods for C++ regexp builtins.\nclass RegExpUtils : public AllStatic {\n public:\n // Last match info accessors.\n static Handle<String> GenericCaptureGetter(\n Isolate* isolate, DirectHandle<RegExpMatchInfo> match_info, int capture,\n bool* ok = nullptr);\n // Checks if the capture group referred to by index |capture| is part of the\n // match.\n static bool IsMatchedCapture(Tagged<RegExpMatchInfo> match_info, int capture);\n\n", "context": "Topic: V8 regexp\nFile: regexp/regexp-utils.h\nChunk: 1/4\nChars: 0-825", "chunk_index": 0, "total_chunks": 4, "start_char": 0, "end_char": 825, "start_line": 1, "end_line": 29, "char_range": "0-825"},
3+
{"doc_id": "d62eb64de1cd378c8ad51439d154465b68b01090", "path": "regexp/regexp-utils.h", "parent_file": "regexp/regexp-utils.h", "parent_id": "36e4aa48dfb01fe5f5d9f9b0c3c9f35a6cbfa2d5", "topic": "V8 regexp", "doc_type": "code", "source": "v8_source", "content": " bool* ok = nullptr);\n // Checks if the capture group referred to by index |capture| is part of the\n // match.\n static bool IsMatchedCapture(Tagged<RegExpMatchInfo> match_info, int capture);\n\n // Last index (RegExp.lastIndex) accessors.\n static V8_WARN_UNUSED_RESULT MaybeDirectHandle<Object> SetLastIndex(\n Isolate* isolate, DirectHandle<JSReceiver> regexp, uint64_t value);\n static V8_WARN_UNUSED_RESULT MaybeDirectHandle<Object> GetLastIndex(\n Isolate* isolate, DirectHandle<JSReceiver> recv);\n\n // ES#sec-regexpexec Runtime Semantics: RegExpExec ( R, S )\n static V8_WARN_UNUSED_RESULT MaybeDirectHandle<JSAny> RegExpExec(\n Isolate* isolate, DirectHandle<JSReceiver> regexp,\n DirectHandle<String> string, DirectHandle<Object> exec);\n\n", "context": "Topic: V8 regexp\nFile: regexp/regexp-utils.h\nChunk: 2/4\nChars: 625-1395", "chunk_index": 1, "total_chunks": 4, "start_char": 625, "end_char": 1395, "start_line": 24, "end_line": 40, "char_range": "625-1395"},
4+
{"doc_id": "803d09e63c0ab9969fb57698405dc3e750052978", "path": "regexp/regexp-utils.h", "parent_file": "regexp/regexp-utils.h", "parent_id": "36e4aa48dfb01fe5f5d9f9b0c3c9f35a6cbfa2d5", "topic": "V8 regexp", "doc_type": "code", "source": "v8_source", "content": "c ( R, S )\n static V8_WARN_UNUSED_RESULT MaybeDirectHandle<JSAny> RegExpExec(\n Isolate* isolate, DirectHandle<JSReceiver> regexp,\n DirectHandle<String> string, DirectHandle<Object> exec);\n\n // Checks whether the given object is an unmodified JSRegExp instance.\n // Neither the object's map, nor its prototype's map, nor any relevant\n // method on the prototype may be modified.\n //\n // Note: This check is limited may only be used in situations where the only\n // relevant property is 'exec'.\n static bool IsUnmodifiedRegExp(Isolate* isolate, DirectHandle<Object> obj);\n\n // ES#sec-advancestringindex\n // AdvanceStringIndex ( S, index, unicode )\n static uint64_t AdvanceStringIndex(Tagged<String> string, uint64_t index,\n bool unicode);\n static V8_WARN_UNUSED_RESULT MaybeDirectHandle<Object> SetAdvancedStringIndex(\n Isolate* isolate, DirectHandle<JSReceiver> regexp,\n DirectHandle<String> string, bool unicode);\n};\n\n", "context": "Topic: V8 regexp\nFile: regexp/regexp-utils.h\nChunk: 3/4\nChars: 1195-2181", "chunk_index": 2, "total_chunks": 4, "start_char": 1195, "end_char": 2181, "start_line": 35, "end_line": 57, "char_range": "1195-2181"},
5+
{"doc_id": "fc849bd5d99edbfa0ac0457d88ed196f33e49329", "path": "regexp/regexp-utils.h", "parent_file": "regexp/regexp-utils.h", "parent_id": "36e4aa48dfb01fe5f5d9f9b0c3c9f35a6cbfa2d5", "topic": "V8 regexp", "doc_type": "code", "source": "v8_source", "content": "icode);\n static V8_WARN_UNUSED_RESULT MaybeDirectHandle<Object> SetAdvancedStringIndex(\n Isolate* isolate, DirectHandle<JSReceiver> regexp,\n DirectHandle<String> string, bool unicode);\n};\n\n} // namespace internal\n} // namespace v8\n\n#endif // V8_REGEXP_REGEXP_UTILS_H_\n", "context": "Topic: V8 regexp\nFile: regexp/regexp-utils.h\nChunk: 4/4\nChars: 1981-2263", "chunk_index": 3, "total_chunks": 4, "start_char": 1981, "end_char": 2263, "start_line": 51, "end_line": 61, "char_range": "1981-2263"},
6+
{"doc_id": "9660262d6b683871824d7ec88d931c3138818e08", "path": "regexp/regexp-result-vector.h", "parent_file": "regexp/regexp-result-vector.h", "parent_id": "77fb896b4aa10b8426b17b865189410d3186d517", "topic": "V8 regexp", "doc_type": "code", "source": "v8_source", "content": "// Copyright 2024 the V8 project authors. All rights reserved.\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\n#ifndef V8_REGEXP_REGEXP_RESULT_VECTOR_H_\n#define V8_REGEXP_REGEXP_RESULT_VECTOR_H_\n\n#include \"src/base/macros.h\"\n#include \"src/common/globals.h\"\n\nnamespace v8 {\nnamespace internal {\n\nclass RegExpResultVectorScope final {\n public:\n explicit RegExpResultVectorScope(Isolate* isolate);\n RegExpResultVectorScope(Isolate* isolate, int size);\n ~RegExpResultVectorScope();\n\n int32_t* Initialize(int size);\n\n int32_t* value() const {\n DCHECK_NOT_NULL(value_);\n return value_;\n }\n\n private:\n Isolate* const isolate_;\n bool is_dynamic_ = false;\n int32_t* value_ = nullptr;\n};\n\nclass RegExpResultVector final : public AllStatic {\n public:\n static int32_t* Allocate(Isolate* isolate, uint32_t size);\n static void Free(Isolate* isolate, int32_t* vector);\n};\n\n} // namespace internal\n} // namespace v8\n\n", "context": "Topic: V8 regexp\nFile: regexp/regexp-result-vector.h\nChunk: 1/2\nChars: 0-977", "chunk_index": 0, "total_chunks": 2, "start_char": 0, "end_char": 977, "start_line": 1, "end_line": 42, "char_range": "0-977"},
7+
{"doc_id": "0c688b356d35f294b4fd35a847ccad9e3f97fb3d", "path": "regexp/regexp-result-vector.h", "parent_file": "regexp/regexp-result-vector.h", "parent_id": "77fb896b4aa10b8426b17b865189410d3186d517", "topic": "V8 regexp", "doc_type": "code", "source": "v8_source", "content": "inal : public AllStatic {\n public:\n static int32_t* Allocate(Isolate* isolate, uint32_t size);\n static void Free(Isolate* isolate, int32_t* vector);\n};\n\n} // namespace internal\n} // namespace v8\n\n#endif // V8_REGEXP_REGEXP_RESULT_VECTOR_H_\n", "context": "Topic: V8 regexp\nFile: regexp/regexp-result-vector.h\nChunk: 2/2\nChars: 777-1022", "chunk_index": 1, "total_chunks": 2, "start_char": 777, "end_char": 1022, "start_line": 33, "end_line": 43, "char_range": "777-1022"},
8+
{"doc_id": "e3bcce34017398b2b4f17ad518a01eaa30952ff6", "path": "regexp/regexp-parser.h", "parent_file": "regexp/regexp-parser.h", "parent_id": "f529896758b0edf35668291b2b0275ce389ecaf7", "topic": "V8 regexp", "doc_type": "code", "source": "v8_source", "content": "// Copyright 2016 the V8 project authors. All rights reserved.\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\n#ifndef V8_REGEXP_REGEXP_PARSER_H_\n#define V8_REGEXP_REGEXP_PARSER_H_\n\n#include \"src/common/assert-scope.h\"\n#include \"src/handles/handles.h\"\n#include \"src/regexp/regexp-flags.h\"\n\nnamespace v8 {\nnamespace internal {\n\nclass String;\nclass Zone;\n\nstruct RegExpCompileData;\n\nclass V8_EXPORT_PRIVATE RegExpParser : public AllStatic {\n public:\n static bool ParseRegExpFromHeapString(Isolate* isolate, Zone* zone,\n DirectHandle<String> input,\n RegExpFlags flags,\n RegExpCompileData* result);\n\n", "context": "Topic: V8 regexp\nFile: regexp/regexp-parser.h\nChunk: 1/2\nChars: 0-771", "chunk_index": 0, "total_chunks": 2, "start_char": 0, "end_char": 771, "start_line": 1, "end_line": 27, "char_range": "0-771"},
9+
{"doc_id": "3f7ebb85f989faed7a8c6aaeda9fee0dadd957bb", "path": "regexp/regexp-parser.h", "parent_file": "regexp/regexp-parser.h", "parent_id": "f529896758b0edf35668291b2b0275ce389ecaf7", "topic": "V8 regexp", "doc_type": "code", "source": "v8_source", "content": "ne,\n DirectHandle<String> input,\n RegExpFlags flags,\n RegExpCompileData* result);\n\n template <class CharT>\n static bool VerifyRegExpSyntax(Zone* zone, uintptr_t stack_limit,\n const CharT* input, int input_length,\n RegExpFlags flags, RegExpCompileData* result,\n const DisallowGarbageCollection& no_gc);\n};\n\n} // namespace internal\n} // namespace v8\n\n#endif // V8_REGEXP_REGEXP_PARSER_H_\n", "context": "Topic: V8 regexp\nFile: regexp/regexp-parser.h\nChunk: 2/2\nChars: 571-1175", "chunk_index": 1, "total_chunks": 2, "start_char": 571, "end_char": 1175, "start_line": 22, "end_line": 38, "char_range": "571-1175"}

0 commit comments

Comments
 (0)