Include visualizer in docs site (#301)

* Include visualizer in docs site

* Fix linting error causing build failure

* Fix static site path

* Trigger docs build when apropos

* Use relative resource paths

* Try different link path

* Use absolute path

* Support pre-aggregated event streams in the UI

* Optionally gzip-compress output

* Speed up CBOR decoding

* Add trace to git

* Add more traces to git

Author: SupernaviX

.gitattributes

@@ -0,0 +1,3 @@
+ui/public/traces/1xsummary.jsonl.gz filter=lfs diff=lfs merge=lfs -text
+ui/public/traces/100xsummary.jsonl.gz filter=lfs diff=lfs merge=lfs -text
+ui/public/traces/10xsummary.jsonl.gz filter=lfs diff=lfs merge=lfs -text

.github/workflows/docs.yaml

@@ -5,12 +5,14 @@ on:
     paths:
       - "**/*.d2"
       - "site/**"
+      - "ui/**"
   push:
     branches:
       - main
     paths:
       - "**/*.d2"
       - "site/**"
+      - "ui/**"
 
 jobs:
   docs-generate-d2-diagrams:
@@ -72,9 +74,43 @@ jobs:
           git commit -m "Auto-generate diagram PNGs [skip ci]"
           git push origin HEAD:${{ github.head_ref || github.ref_name }}
 
+  viz-build:
+    name: "Build Visualizer"
+    runs-on: ubuntu-22.04
+    steps:
+      - name: 📥 Checkout repository
+        uses: actions/checkout@v4
+        with:
+          lfs: true
+          fetch-depth: 2
+      
+      - name: 🛠️ Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: "yarn"
+          cache-dependency-path: ./ui/yarn.lock 
+
+      - name: 📦 Install dependencies
+        working-directory: ui
+        run: yarn install
+  
+      - name: 🏗️ Build visualizer
+        working-directory: ui
+        run: |
+          yarn build
+
+      - name: 🚢 Upload visualizer static site
+        id: upload_viz
+        uses: actions/upload-artifact@v4
+        with:
+          name: visualizer
+          path: ui/dist
+
   docs-build:
     name: "Build"
     runs-on: ubuntu-22.04
+    needs: viz-build
     outputs:
       has_changes: ${{ steps.check_changes.outputs.has_changes }}
     steps:
@@ -83,11 +119,12 @@ jobs:
         with:
           fetch-depth: 2
 
-      - name: Check for site changes
+      - name: Check for site or visualizer changes
         id: check_changes
         run: |
           SITE_CHANGES=$(git diff --name-only HEAD^ HEAD -- site/ || true)
-          if [ -z "$SITE_CHANGES" ]; then
+          VIZ_CHANGES=$(git diff --name-only HEAD^ HEAD -- ui/ || true)
+          if [ -z "$SITE_CHANGES" ] && [ -z "$VIZ_CHANGES" ]; then
             echo "No changes in site directory"
             echo "has_changes=false" >> $GITHUB_OUTPUT
           else
@@ -110,6 +147,13 @@ jobs:
         working-directory: site
         run: yarn install
 
+      - name: 👁️ Unpack visualizer
+        if: steps.check_changes.outputs.has_changes == 'true'
+        uses: actions/download-artifact@v4
+        with:
+          name: visualizer
+          path: site/static/visualizer
+
       - name: 🏗️ Build Docusaurus site
         if: steps.check_changes.outputs.has_changes == 'true'
         working-directory: site

sim-rs/Cargo.lock

@@ -7,6 +7,7 @@ rust-version = "1.82"
 
 [dependencies]
 anyhow = "1"
+async-compression = { version = "0.4", features = ["tokio", "gzip"] }
 average = "0.16"
 clap = { version = "4", features = ["derive"] }
 ctrlc = "3"

sim-rs/sim-cli/Cargo.toml

@@ -1,9 +1,12 @@
 use std::{
     collections::{BTreeMap, BTreeSet},
     path::PathBuf,
+    pin::Pin,
 };
 
+use aggregate::TraceAggregator;
 use anyhow::Result;
+use async_compression::tokio::write::GzipEncoder;
 use average::Variance;
 use itertools::Itertools as _;
 use pretty_bytes_rust::{pretty_bytes, PrettyBytesOptions};
@@ -16,15 +19,19 @@ use sim_core::{
 };
 use tokio::{
     fs::{self, File},
-    io::{AsyncWriteExt as _, BufWriter},
+    io::{AsyncWrite, AsyncWriteExt as _, BufWriter},
     sync::mpsc,
 };
 use tracing::{info, info_span};
 
+mod aggregate;
+
 type InputBlockId = sim_core::model::InputBlockId<Node>;
 type EndorserBlockId = sim_core::model::EndorserBlockId<Node>;
 type VoteBundleId = sim_core::model::VoteBundleId<Node>;
 
+type TraceSink = Pin<Box<dyn AsyncWrite + Send + Sync + 'static>>;
+
 #[derive(Clone, Serialize)]
 struct OutputEvent {
     time_s: Timestamp,
@@ -44,6 +51,7 @@ pub struct EventMonitor {
     maximum_eb_age: u64,
     events_source: mpsc::UnboundedReceiver<(Event, Timestamp)>,
     output_path: Option<PathBuf>,
+    aggregate: bool,
 }
 
 impl EventMonitor {
@@ -67,6 +75,7 @@ impl EventMonitor {
             maximum_eb_age: config.max_eb_age,
             events_source,
             output_path,
+            aggregate: config.aggregate_events,
         }
     }
 
@@ -125,9 +134,27 @@ impl EventMonitor {
             }
         }
 
-        let mut output = match self.output_path {
-            Some(ref path) => {
-                let file = File::create(path).await?;
+        let mut output = match self.output_path.as_mut() {
+            Some(path) => {
+                let file = File::create(&path).await?;
+
+                let mut gzipped = false;
+                if path
+                    .extension()
+                    .and_then(|e| e.to_str())
+                    .is_some_and(|ext| ext == "gz")
+                {
+                    path.set_extension("");
+                    gzipped = true;
+                }
+
+                let file: TraceSink = if gzipped {
+                    let encoder = GzipEncoder::new(file);
+                    Box::pin(BufWriter::new(encoder))
+                } else {
+                    Box::pin(BufWriter::new(file))
+                };
+
                 let format = if path
                     .extension()
                     .and_then(|e| e.to_str())
@@ -137,9 +164,14 @@ impl EventMonitor {
                 } else {
                     OutputFormat::JsonStream
                 };
-                OutputTarget::EventStream {
-                    format,
-                    file: BufWriter::new(file),
+                if self.aggregate {
+                    OutputTarget::AggregatedEventStream {
+                        aggregation: TraceAggregator::new(),
+                        format,
+                        file,
+                    }
+                } else {
+                    OutputTarget::EventStream { format, file }
                 }
             }
             None => OutputTarget::None,
@@ -576,16 +608,30 @@ fn compute_stats<Iter: IntoIterator<Item = f64>>(data: Iter) -> Stats {
 }
 
 enum OutputTarget {
+    AggregatedEventStream {
+        aggregation: TraceAggregator,
+        format: OutputFormat,
+        file: TraceSink,
+    },
     EventStream {
         format: OutputFormat,
-        file: BufWriter<File>,
+        file: TraceSink,
     },
     None,
 }
 
 impl OutputTarget {
     async fn write(&mut self, event: OutputEvent) -> Result<()> {
         match self {
+            Self::AggregatedEventStream {
+                aggregation,
+                format,
+                file,
+            } => {
+                if let Some(summary) = aggregation.process(event) {
+                    Self::write_line(*format, file, summary).await?;
+                }
+            }
             Self::EventStream { format, file } => {
                 Self::write_line(*format, file, event).await?;
             }
@@ -594,9 +640,9 @@ impl OutputTarget {
         Ok(())
     }
 
-    async fn write_line<T: Serialize>(
+    async fn write_line<T: Serialize, W: AsyncWrite + Unpin>(
         format: OutputFormat,
-        file: &mut BufWriter<File>,
+        file: &mut W,
         event: T,
     ) -> Result<()> {
         match format {
@@ -615,6 +661,16 @@ impl OutputTarget {
 
     async fn flush(self) -> Result<()> {
         match self {
+            Self::AggregatedEventStream {
+                aggregation,
+                format,
+                mut file,
+            } => {
+                if let Some(summary) = aggregation.finish() {
+                    Self::write_line(format, &mut file, summary).await?;
+                }
+                file.flush().await?;
+            }
             Self::EventStream { mut file, .. } => {
                 file.flush().await?;
             }