Chaos features (#45)

* Added node endpoint for crashing individual actors

Untested though

Also make the rpc_client generation makefile code easier to run

* Actor crashing implemented

* Removing clone for NodeHandle

* Client side code done for dupl, loss

ChaosManager at Actor TBD

Co-authored-by: Brian Sajeev <Brian-1402@users.noreply.github.com>

* Actor side ChaosManager completed

Co-authored-by: Brian Sajeev <Brian-1402@users.noreply.github.com>

* WIP review changes

* Global Chaos Implemented (Untested)

Separate Chaos Endpoints for cleaner chaos request types

Probability & Factor Toml validation

Co-authored-by: Brian Sajeev <Brian-1402@users.noreply.github.com>

* added delay chaos

* Actor Restarts implemented (Untested)

Chaos Refactored into apply/revert types

* Stop chaos endpoint added (Untested)

* Fixed merge compile errors, incomplete UnsetMsgDelay endpoint

Also formatting and clippy satisfaction

* Disable delay chaos added (Untested)

* Added paxos to examples

* debugged delay

* Merged master (#53)

* added docs workflow

* Update docs.yml

* Update docs.yml #skipci

* Update docs.yml [skip ci]

* Update docs.yml [skip ci]

* added browser based job controller

* updated ui

* formatting

* Update api (#49)

* updated generated api clients

* clippy

* Added #actor macro to register the actor (#50)

* updated generated api clients

* clippy

* updated api to get list of registerd apis

* wip ui

* added how_to.md

* added register actor macro

* added paxos to ci

* merged with master

* fixed makefile

* formatting

---------

Co-authored-by: Aman-Hassan <amanhassan2003@gmail.com>
Co-authored-by: Brian Sajeev <Brian-1402@users.noreply.github.com>
Co-authored-by: Satyam Jay <satyamjay030@gmail.com>

Author: 3 people

.github/workflows/rust.yml

@@ -57,3 +57,16 @@ jobs:
               echo "make job failed"
               exit $status
           fi
+
+    - name: Run Paxos
+      run: |
+          cd examples/paxos
+          make node &
+          sleep 1
+          
+          timeout 6s make job || echo "make job terminated after timeout"
+          status=$?
+          if [ $status -ne 0 ] && [ $status -ne 124 ]; then
+              echo "make job failed"
+              exit $status
+          fi

Cargo.lock

@@ -7,6 +7,7 @@ members = [
     "generic_nctrl",
     "generic_jctrl",
     "macros",
+    "examples/paxos",
     "examples/ping_pong",
     "examples/read_write_server",
 ]
@@ -17,7 +18,7 @@ exclude = [
 resolver = "2"
 
 [workspace.package]
-rust-version = "1.86"
+rust-version = "1.89"
 edition = "2024"
 license = "MIT OR Apache-2.0"
 authors = ["Satyam Jay"]

Cargo.toml

@@ -1,4 +1,4 @@
-gen_http_client:
+gen_http_client: kill_node
 	echo "Running demo server..."
 	- pkill reactor_nctrl
 
@@ -14,3 +14,21 @@ gen_http_client:
 		-g typescript-axios -o ./reactor-dashboard/api-client/src --additional-properties=packageName=reactor-client
 
 	- pkill reactor_nctrl
+
+
+kill_node:
+	@echo "Killing process on port 3000 if any..."
+	@lsof -ti :3000 | xargs --no-run-if-empty kill
+
+node: kill_node
+	@echo "Running demo server..."
+	cargo run --features swagger --bin reactor_nctrl -- --port 3000 /tmp
+
+generate:
+	@echo "Generating client"
+	openapi-generator-cli generate -i http://localhost:3000/api-doc/openapi.json \
+		-g rust -o rpc_client/ --additional-properties=packageName=reactor-client
+
+	openapi-generator-cli generate -i http://localhost:3000/api-doc/openapi.json \
+		-g typescript-axios -o ./reactor-dashboard/api-client/src --additional-properties=packageName=reactor-client
+

Makefile

@@ -20,6 +20,7 @@ tracing = {version = "0.1.41", features = ["std", "attributes"] }
 tracing-subscriber = "0.3.19"
 tracing-shared = "0.1.5"
 criterion = "0.6.0"
+rand = "0.9.2"
 serde_json = "1.0.143"
 inventory = "0.3.21"
 reactor-macros = { path = "../macros" }

actor/Cargo.toml

@@ -0,0 +1,59 @@
+use rand::Rng;
+
+#[derive(Debug)]
+pub struct ChaosManager {
+    pub msg_loss_probability: Option<f32>,
+    pub msg_duplication_factor: Option<u32>,
+    pub msg_duplication_probability: Option<f32>,
+}
+
+impl ChaosManager {
+    pub fn new() -> Self {
+        ChaosManager {
+            msg_loss_probability: None,
+            msg_duplication_factor: None,
+            msg_duplication_probability: None,
+        }
+    }
+
+    pub fn set_msg_loss(&mut self, probability: f32) {
+        self.msg_loss_probability = Some(probability);
+    }
+
+    pub fn set_msg_duplication(&mut self, factor: u32, probability: f32) {
+        self.msg_duplication_factor = Some(factor);
+        self.msg_duplication_probability = Some(probability);
+    }
+
+    #[allow(dead_code)]
+    pub fn unset_msg_loss(&mut self) {
+        self.msg_loss_probability = None;
+    }
+
+    #[allow(dead_code)]
+    pub fn unset_msg_duplication(&mut self) {
+        self.msg_duplication_factor = None;
+        self.msg_duplication_probability = None;
+    }
+
+    pub fn apply_chaos<T: Clone>(&self, msg: T) -> Vec<T> {
+        let mut rng = rand::rng();
+
+        if self
+            .msg_loss_probability
+            .is_some_and(|p| rng.random_bool(p as f64))
+        {
+            return vec![];
+        }
+
+        if let (Some(factor), Some(prob)) = (
+            self.msg_duplication_factor,
+            self.msg_duplication_probability,
+        ) && rng.random_bool(prob as f64)
+        {
+            return vec![msg; factor as usize];
+        }
+
+        vec![msg]
+    }
+}