automate the generation of the memory flamegraph

Signed-off-by: William Johnson <[email protected]>

Author: johnsonw

.github/workflows/generate-memory-flamegraph.yml

@@ -0,0 +1,116 @@
+on:
+  push:
+    branches:
+      - main
+
+  pull_request:
+    types: [opened, reopened, edited, synchronize]
+
+defaults:
+  run:
+    shell: bash -eux {0}
+
+jobs:
+  generate_memory_flamegraph:
+    name: Create flamegraph for scrape_flamegraph
+    permissions:
+      checks: write
+      pull-requests: write
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/checkout@v4
+        with:
+          repository: rust-lang/rustc-demangle
+          path: rustc-demangle
+
+      - uses: actions/checkout@v4
+        with:
+          repository: KDE/heaptrack
+          path: heaptrack
+
+      - uses: actions/checkout@v4
+        with:
+          repository: brendangregg/FlameGraph
+          path: FlameGraph
+
+      - name: Install Rust
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          rustflags: ""
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y \
+            build-essential libssl-dev pkg-config curl git \
+            cmake g++ libboost-dev libboost-iostreams-dev \
+            libboost-program-options-dev libdw-dev libelf-dev \
+            elfutils libunwind-dev zlib1g-dev binutils-dev \
+            libiberty-dev libdebuginfod-dev libboost-system-dev \
+            libboost-filesystem-dev libboost-thread-dev libboost-regex-dev
+          cargo install rustfilt
+
+      - name: Build and install librustc_demangle.so
+        run: |
+          cd rustc-demangle
+          cargo build -p rustc-demangle-capi --release
+          sudo mv target/release/librustc_demangle.so /usr/lib
+
+      - name: Build and install heaptrack
+        run: |
+          cd heaptrack
+          mkdir build && cd build
+
+          cmake .. -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_INSTALL_PREFIX=/usr/local -DHEAPTRACK_USE_LIBUNWIND=ON -DHEAPTRACK_BUILD_PRINT_TRACES=ON
+
+          make -j$(nproc)
+          sudo make install
+
+      - name: Build flamegraph
+        run: |
+           cargo build --profile profiling --test flamegraph_test
+        env:
+          RUSTFLAGS: "-C force-frame-pointers=yes -C symbol-mangling-version=v0"
+          CARGO_PROFILE_RELEASE_DEBUG: true
+
+      - name: Build heaptrack data
+        run: |
+          export TEST_BINARY=$(find "$GITHUB_WORKSPACE/target/profiling/deps" -name "flamegraph_test-*" -type f -executable | head -1)
+
+          if [ -z "$TEST_BINARY" ]; then
+            echo "Error: Could not find test binary"
+            exit 1
+          fi
+
+          "$TEST_BINARY" profile_with_flamegraph --ignored  --no-capture | head -100
+
+          heaptrack --output heaptrack-data "$TEST_BINARY" --ignored profile_with_flamegraph
+        env:
+          CARGO_MANIFEST_DIR: "$GITHUB_WORKSPACE/lustrefs-exporter"
+
+      - name: Generate flamegraph
+        run: |
+          heaptrack_print --version
+          HEAPTRACK_FILE=$(ls heaptrack-data.* | head -1)
+
+          echo "Writing flamegraph data to flamegraph-data.txt"
+          heaptrack_print "$HEAPTRACK_FILE" -F flamegraph-data.txt > /dev/null
+          echo "Done writing flamegraph data to flamegraph-data.txt: $?"
+
+          rustfilt < flamegraph-data.txt > demangled-flamegraph.txt
+          ls -lah demangled-flamegraph.txt
+          awk '$NF > 1000' demangled-flamegraph.txt > significant_stacks.txt
+          ls -lah significant_stacks.txt
+          FlameGraph/flamegraph.pl --colors mem --countname allocations < significant_stacks.txt > flamegraph.svg
+          ls -lah flamegraph.svg
+
+      - name: Upload flamegraph
+        uses: actions/upload-artifact@v4
+        with:
+          name: flamegraph
+          path: |
+            flamegraph.svg
+            cpu-flamegraph.svg
+            heaptrack-data.*

.github/workflows/scrape-memory-metrics-bench-flame.yml

@@ -17,6 +17,7 @@ regex = { version = "1", default-features = false, features = [
   "std",
   "perf-dfa-full",
 ] }
+rustc-demangle.workspace = true
 serde = { version = "1", features = ["derive"] }
 thiserror.workspace = true
 tokio = { workspace = true, features = [

Cargo.lock

@@ -1,19 +1,14 @@
-// #[global_allocator]
-// static ALLOC: dhat::Alloc = dhat::Alloc;
-
 use commandeer_test::commandeer;
 use pprof::ProfilerGuardBuilder;
 use serial_test::serial;
-use tokio::task::JoinSet;
 use std::time::{Duration, Instant};
+use tokio::task::JoinSet;
 
 #[commandeer(Replay, "lctl", "lnetctl")]
 #[tokio::test]
 #[serial]
+#[ignore]
 async fn profile_with_flamegraph() {
-    // Start memory profiling
-    //let _mem_profiler = dhat::Profiler::new_heap();
-
     // Start CPU profiling
     let cpu_guard = ProfilerGuardBuilder::default()
         .frequency(100)
@@ -35,9 +30,7 @@ async fn profile_with_flamegraph() {
     tokio::time::sleep(Duration::from_millis(100)).await;
 
     // Run your load test once
-    println!("Running load test...");
-    let duration = load_test_concurrent(1, 1).await;
-    println!("Load test completed in {:?}", duration);
+    let _duration = load_test_concurrent(10, 10).await;
 
     // Generate CPU flamegraph
     if let Ok(report) = cpu_guard.report().build() {
@@ -49,11 +42,6 @@ async fn profile_with_flamegraph() {
 
     // dhat automatically saves to dhat-heap.json when _mem_profiler drops
     println!("Memory profile saved to dhat-heap.json");
-
-    // Get memory stats
-    // let stats = dhat::HeapStats::get();
-    // println!("Peak memory: {} MB", stats.max_bytes / 1_000_000);
-    // println!("Total allocations: {}", stats.total_blocks);
 }
 
 // Create a single request using `oneshot`. This is equivalent to hitting the