diff --git a/.github/workflows/automated-experiment-result-checker.yml b/.github/workflows/automated-experiment-result-checker.yml new file mode 100644 index 000000000..d9990fe6e --- /dev/null +++ b/.github/workflows/automated-experiment-result-checker.yml @@ -0,0 +1,66 @@ +--- +name: "Automated Experiment Result Checker" + +# yamllint disable-line rule:truthy +on: + pull_request: + types: [opened, reopened, synchronize] + +concurrency: + group: ${{ github.ref }}-automated-experiment-result-checker + cancel-in-progress: true + +permissions: + contents: write + pull-requests: write +jobs: + automated-experiment-result-checker: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + + - name: Check for updated experiment result graphs + run: | + set -e + cd "$(git rev-parse --show-toplevel)" + + # TODO: Include lower bound windup experiment once we have a way to make it run in a reasonable time. + # Find all PNGs, excluding those with "windup" in their filename + mapfile -t all_pngs < <(find experiments/results/main_graphs experiments/results/throughput_graphs experiments/results/duration_graphs -type f -name '*.png' ! -name '*windup*.png' | sort) + + # Find all changed PNGs in the latest commit + mapfile -t changed_pngs < <(git diff --name-only --diff-filter=AM HEAD~1..HEAD | grep -E '^experiments/results/(main_graphs|throughput_graphs|duration_graphs)/.*\.png$' | grep -v windup | sort) + + # Report any PNGs that are not updated in the latest commit + declare -a not_updated=() + for file in "${all_pngs[@]}"; do + if ! printf "%s\n" "${changed_pngs[@]}" | grep -qx "$file"; then + not_updated+=("$file") + fi + done + + if [ ${#not_updated[@]} -gt 0 ]; then + echo "❌ The following result graph PNG files have NOT been updated in the latest commit:" + for f in "${not_updated[@]}"; do + echo " - $f" + done + echo "" + echo "Every commit must update all non-windup experiment result graphs. You may be missing updates." + echo "Run:" + echo "" + echo " cd experiments" + echo " bundle install" + echo " bundle exec ruby run_all_experiments.rb" + echo "" + echo "Commit the updated graphs to resolve this check." + exit 1 + fi + + echo "✅ All non-windup experiment result graphs are up to date for this commit!" + + + diff --git a/experiments/duration-one_of_many_services_latency_degradation.png b/experiments/duration-one_of_many_services_latency_degradation.png deleted file mode 100644 index 0c31a6928..000000000 Binary files a/experiments/duration-one_of_many_services_latency_degradation.png and /dev/null differ diff --git a/experiments/duration-one_of_many_services_latency_degradation_adaptive.png b/experiments/duration-one_of_many_services_latency_degradation_adaptive.png deleted file mode 100644 index 61a54b73a..000000000 Binary files a/experiments/duration-one_of_many_services_latency_degradation_adaptive.png and /dev/null differ diff --git a/experiments/example_output.png b/experiments/example_output.png deleted file mode 100644 index 0976a711c..000000000 Binary files a/experiments/example_output.png and /dev/null differ diff --git a/experiments/example_with_circuit_breaker.rb b/experiments/example_with_circuit_breaker.rb deleted file mode 100644 index d89c0af7e..000000000 --- a/experiments/example_with_circuit_breaker.rb +++ /dev/null @@ -1,93 +0,0 @@ -# frozen_string_literal: true - -$LOAD_PATH.unshift(File.expand_path("../lib", __dir__)) - -require "semian" -require_relative "mock_service" -require_relative "experimental_resource" - -# Create the mock service -service = Semian::Experiments::MockService.new( - endpoints_count: 200, - min_latency: 0.01, - max_latency: 10, - distribution: { - type: :log_normal, - mean: 1, - std_dev: 0.1, - }, - error_rate: 0.01, # 1% baseline error rate - timeout: 5, # 5 seconds timeout -) - -# Create the Semian adapter for the service -resource = Semian::Experiments::ExperimentalResource.new( - name: "protected_service", - service: service, - semian: { - success_threshold: 2, - error_threshold: 3, - error_threshold_timeout: 20, - error_timeout: 15, - bulkhead: false, - }, -) - -outcomes = {} - -done = false - -Thread.new do - until done - sleep(0.1) # Don't send more than 10 requests per second - current_sec = outcomes[Time.now.to_i] ||= { - success: 0, - circuit_open: 0, - error: 0, - } - begin - resource.request(rand(service.endpoints_count)) - puts "✓ Success" - current_sec[:success] += 1 - rescue Semian::Experiments::ExperimentalResource::CircuitOpenError => e - puts "⚡ Circuit Open - #{e.message}" - current_sec[:circuit_open] += 1 - rescue Semian::Experiments::ExperimentalResource::RequestError, Semian::Experiments::ExperimentalResource::TimeoutError => e - puts "✗ Error" - current_sec[:error] += 1 - end - end -end - -sleep 10 - -puts "Setting error rate to 0.5" -service.set_error_rate(0.5) - -sleep 10 - -puts "Resetting error rate to 0.01" -service.set_error_rate(0.01) - -sleep 10 - -done = true - -puts "Generating graph showing success, circuit open, and error rates over time, (bucketed by 1 second)..." -require "gruff" - -graph = Gruff::Line.new -graph.title = "Outcomes" -graph.x_axis_label = "Time" -graph.y_axis_label = "Count" - -graph.hide_dots = true -graph.line_width = 3 - -graph.data("Success", outcomes.map { |_, data| data[:success] }) -graph.data("Circuit Open", outcomes.map { |_, data| data[:circuit_open] }) -graph.data("Error", outcomes.map { |_, data| data[:error] }) - -graph.write("example_output.png") - -puts "Graph saved to outcomes.png" diff --git a/experiments/test_helpers.rb b/experiments/experiment_helpers.rb similarity index 91% rename from experiments/test_helpers.rb rename to experiments/experiment_helpers.rb index 10e1a9e4b..d0b927a0e 100644 --- a/experiments/test_helpers.rb +++ b/experiments/experiment_helpers.rb @@ -2,8 +2,9 @@ module Semian module Experiments - # Test runner for circuit breaker experiments (both adaptive and classic) + # Experiment runner for circuit breaker experiments (both adaptive and classic) # Handles all the common logic: service creation, threading, monitoring, analysis, and visualization + require "fileutils" class DegradationPhase attr_reader :healthy, :error_rate, :latency @@ -14,11 +15,11 @@ def initialize(healthy: nil, error_rate: nil, latency: nil) end end - class CircuitBreakerTestRunner - attr_reader :test_name, :resource_name, :degradation_phases, :phase_duration, :graph_title, :graph_filename, :service_count, :target_service + class CircuitBreakerExperimentRunner + attr_reader :experiment_name, :resource_name, :degradation_phases, :phase_duration, :graph_title, :graph_filename, :service_count, :target_service def initialize( - test_name:, + experiment_name:, resource_name:, degradation_phases:, phase_duration:, @@ -32,7 +33,7 @@ def initialize( graph_bucket_size: nil, base_error_rate: nil ) - @test_name = test_name + @experiment_name = experiment_name @resource_name = resource_name @degradation_phases = degradation_phases @phase_duration = phase_duration @@ -40,10 +41,16 @@ def initialize( @semian_config = semian_config @is_adaptive = semian_config[:adaptive_circuit_breaker] == true @graph_filename = graph_filename || "#{resource_name}.png" + @main_results_path = File.join(File.dirname(__FILE__), "results/main_graphs") + @duration_results_path = File.join(File.dirname(__FILE__), "results/duration_graphs") + @throughput_results_path = File.join(File.dirname(__FILE__), "results/throughput_graphs") + FileUtils.mkdir_p(@main_results_path) unless File.directory?(@main_results_path) + FileUtils.mkdir_p(@duration_results_path) unless File.directory?(@duration_results_path) + FileUtils.mkdir_p(@throughput_results_path) unless File.directory?(@throughput_results_path) @num_threads = num_threads @requests_per_second_per_thread = requests_per_second_per_thread @x_axis_label_interval = x_axis_label_interval || phase_duration - @test_duration = degradation_phases.length * phase_duration + @experiment_duration = degradation_phases.length * phase_duration @service_count = service_count @target_service = nil @graph_bucket_size = graph_bucket_size || (@is_adaptive ? 10 : 1) @@ -232,12 +239,12 @@ def subscribe_to_state_changes end def execute_phases - puts "\n=== #{@test_name} (ADAPTIVE) ===" + puts "\n=== #{@experiment_name} (ADAPTIVE) ===" puts "Error rate: #{@degradation_phases.map { |r| r.error_rate ? "#{(r.error_rate * 100).round(1)}%" : "N/A" }.join(" -> ")}" puts "Latency: #{@degradation_phases.map { |r| r.latency ? "#{(r.latency * 1000).round(1)}ms" : "N/A" }.join(" -> ")}" puts "Phase duration: #{@phase_duration} seconds (#{(@phase_duration / 60.0).round(1)} minutes) per phase" - puts "Duration: #{@test_duration} seconds (#{(@test_duration / 60.0).round(1)} minutes)" - puts "Starting test...\n" + puts "Duration: #{@experiment_duration} seconds (#{(@experiment_duration / 60.0).round(1)} minutes)" + puts "Starting experiment...\n" @start_time = Time.now @@ -281,7 +288,7 @@ def wait_for_completion end def generate_analysis - puts "\n\n=== Test Complete ===" + puts "\n\n=== Experiment Complete ===" puts "Actual duration: #{(@end_time - @start_time).round(2)} seconds" puts "\nGenerating analysis..." @@ -306,7 +313,7 @@ def display_summary_statistics def display_time_based_analysis bucket_size = @phase_duration - num_buckets = (@test_duration / bucket_size.to_f).ceil + num_buckets = (@experiment_duration / bucket_size.to_f).ceil puts "\n=== Time-Based Analysis (#{bucket_size}-second buckets) ===" (0...num_buckets).each do |bucket_idx| @@ -351,7 +358,7 @@ def display_thread_timing_statistics avg_utilization = (avg_thread_time / total_wall_time * 100) puts "Total threads: #{@thread_timings.size}" - puts "Test wall clock duration: #{total_wall_time.round(2)}s" + puts "Experiment wall clock duration: #{total_wall_time.round(2)}s" puts "\nTime spent making requests per thread:" puts " Min: #{min_thread_time.round(2)}s" puts " Max: #{max_thread_time.round(2)}s" @@ -450,7 +457,7 @@ def generate_visualization # Aggregate data into buckets for detailed visualization bucket_size = @graph_bucket_size - num_buckets = (@test_duration / bucket_size.to_f).ceil + num_buckets = (@experiment_duration / bucket_size.to_f).ceil bucketed_data = [] (0...num_buckets).each do |bucket_idx| @@ -503,8 +510,9 @@ def generate_visualization add_state_transition_markers(graph, bucketed_data, bucket_size, num_buckets) end - graph.write(@graph_filename) - puts "Graph saved to #{@graph_filename}" + main_graph_path = File.join(@main_results_path, @graph_filename) + graph.write(main_graph_path) + puts "Graph saved to #{main_graph_path}" # Generate duration graph duration_graph = Gruff::Line.new(1400) @@ -518,8 +526,9 @@ def generate_visualization duration_graph.data("Total Request Duration", bucketed_data.map { |d| d[:sum_request_duration] }) duration_filename = @graph_filename.sub(%r{([^/]+)$}, 'duration-\1') - duration_graph.write(duration_filename) - puts "Duration graph saved to #{duration_filename}" + duration_graph_path = File.join(@duration_results_path, duration_filename) + duration_graph.write(duration_graph_path) + puts "Duration graph saved to #{duration_graph_path}" # Generate throughput graph throughput_graph = Gruff::Line.new(1400) @@ -533,18 +542,19 @@ def generate_visualization throughput_graph.data("Total Request Throughput", bucketed_data.map { |d| d[:throughput] }) throughput_filename = @graph_filename.sub(%r{([^/]+)$}, 'throughput-\1') - throughput_graph.write(throughput_filename) - puts "Throughput graph saved to #{throughput_filename}" + throughput_graph_path = File.join(@throughput_results_path, throughput_filename) + throughput_graph.write(throughput_graph_path) + puts "Throughput graph saved to #{throughput_graph_path}" end def add_state_transition_markers(graph, bucketed_data, bucket_size, num_buckets) return if @state_transitions.empty? - test_start = @outcomes.keys[0] + experiment_start = @outcomes.keys[0] @state_transitions.each_with_index do |transition, idx| # Calculate which bucket this transition falls into - elapsed = transition[:timestamp] - test_start + elapsed = transition[:timestamp] - experiment_start bucket_idx = (elapsed / bucket_size).to_i next if bucket_idx < 0 || bucket_idx >= num_buckets diff --git a/experiments/test_error_spike_100.rb b/experiments/experiments/experiment_error_spike_100.rb similarity index 58% rename from experiments/test_error_spike_100.rb rename to experiments/experiments/experiment_error_spike_100.rb index c20afb77a..8fea28188 100644 --- a/experiments/test_error_spike_100.rb +++ b/experiments/experiments/experiment_error_spike_100.rb @@ -1,15 +1,15 @@ # frozen_string_literal: true -$LOAD_PATH.unshift(File.expand_path("../lib", __dir__)) +$LOAD_PATH.unshift(File.expand_path("../../lib", __dir__)) require "semian" -require_relative "mock_service" -require_relative "experimental_resource" -require_relative "test_helpers" +require_relative "../mock_service" +require_relative "../experimental_resource" +require_relative "../experiment_helpers" -# Sudden error spike test: 1% -> 100% -> 1% -runner = Semian::Experiments::CircuitBreakerTestRunner.new( - test_name: "Sudden Error Spike Test (Classic) - 100% for 20 seconds", +# Sudden error spike experiment: 1% -> 100% -> 1% +runner = Semian::Experiments::CircuitBreakerExperimentRunner.new( + experiment_name: "Sudden Error Spike Experiment (Classic) - 100% for 20 seconds", resource_name: "protected_service_sudden_error_spike_100", degradation_phases: [Semian::Experiments::DegradationPhase.new(healthy: true)] * 3 + [Semian::Experiments::DegradationPhase.new(error_rate: 1.00)] + @@ -22,7 +22,7 @@ error_timeout: 15, bulkhead: false, }, - graph_title: "Sudden Error Spike Test (Classic) - 100% for 20 seconds", + graph_title: "Sudden Error Spike Experiment (Classic) - 100% for 20 seconds", graph_filename: "sudden_error_spike_100.png", x_axis_label_interval: 60, ) diff --git a/experiments/test_error_spike_100_adaptive.rb b/experiments/experiments/experiment_error_spike_100_adaptive.rb similarity index 55% rename from experiments/test_error_spike_100_adaptive.rb rename to experiments/experiments/experiment_error_spike_100_adaptive.rb index db7739d9d..c137698c2 100644 --- a/experiments/test_error_spike_100_adaptive.rb +++ b/experiments/experiments/experiment_error_spike_100_adaptive.rb @@ -1,15 +1,15 @@ # frozen_string_literal: true -$LOAD_PATH.unshift(File.expand_path("../lib", __dir__)) +$LOAD_PATH.unshift(File.expand_path("../../lib", __dir__)) require "semian" -require_relative "mock_service" -require_relative "experimental_resource" -require_relative "test_helpers" +require_relative "../mock_service" +require_relative "../experimental_resource" +require_relative "../experiment_helpers" -# Sudden error spike test: 1% -> 100% -> 1% -runner = Semian::Experiments::CircuitBreakerTestRunner.new( - test_name: "Sudden Error Spike Test (Adaptive) - 100% for 20 seconds", +# Sudden error spike experiment: 1% -> 100% -> 1% +runner = Semian::Experiments::CircuitBreakerExperimentRunner.new( + experiment_name: "Sudden Error Spike Experiment (Adaptive) - 100% for 20 seconds", resource_name: "protected_service_sudden_error_spike_100_adaptive", degradation_phases: [Semian::Experiments::DegradationPhase.new(healthy: true)] * 3 + [Semian::Experiments::DegradationPhase.new(error_rate: 1.00)] + @@ -19,7 +19,7 @@ adaptive_circuit_breaker: true, bulkhead: false, }, - graph_title: "Sudden Error Spike Test (Adaptive) - 100% for 20 seconds", + graph_title: "Sudden Error Spike Experiment (Adaptive) - 100% for 20 seconds", graph_filename: "sudden_error_spike_100_adaptive.png", x_axis_label_interval: 60, ) diff --git a/experiments/test_error_spike_20.rb b/experiments/experiments/experiment_error_spike_20.rb similarity index 58% rename from experiments/test_error_spike_20.rb rename to experiments/experiments/experiment_error_spike_20.rb index 3f7eeda4e..f16b9a8db 100644 --- a/experiments/test_error_spike_20.rb +++ b/experiments/experiments/experiment_error_spike_20.rb @@ -1,15 +1,15 @@ # frozen_string_literal: true -$LOAD_PATH.unshift(File.expand_path("../lib", __dir__)) +$LOAD_PATH.unshift(File.expand_path("../../lib", __dir__)) require "semian" -require_relative "mock_service" -require_relative "experimental_resource" -require_relative "test_helpers" +require_relative "../mock_service" +require_relative "../experimental_resource" +require_relative "../experiment_helpers" -# Sudden error spike test: 1% -> 20% -> 1% -runner = Semian::Experiments::CircuitBreakerTestRunner.new( - test_name: "Sudden Error Spike Test (Classic) - 20% for 20 seconds", +# Sudden error spike experiment: 1% -> 20% -> 1% +runner = Semian::Experiments::CircuitBreakerExperimentRunner.new( + experiment_name: "Sudden Error Spike Experiment (Classic) - 20% for 20 seconds", resource_name: "protected_service_sudden_error_spike_20", degradation_phases: [Semian::Experiments::DegradationPhase.new(healthy: true)] * 3 + [Semian::Experiments::DegradationPhase.new(error_rate: 0.20)] + @@ -22,7 +22,7 @@ error_timeout: 15, bulkhead: false, }, - graph_title: "Sudden Error Spike Test (Classic) - 20% for 20 seconds", + graph_title: "Sudden Error Spike Experiment (Classic) - 20% for 20 seconds", graph_filename: "sudden_error_spike_20.png", x_axis_label_interval: 60, ) diff --git a/experiments/test_error_spike_20_adaptive.rb b/experiments/experiments/experiment_error_spike_20_adaptive.rb similarity index 56% rename from experiments/test_error_spike_20_adaptive.rb rename to experiments/experiments/experiment_error_spike_20_adaptive.rb index b9072e878..51245823c 100644 --- a/experiments/test_error_spike_20_adaptive.rb +++ b/experiments/experiments/experiment_error_spike_20_adaptive.rb @@ -1,15 +1,15 @@ # frozen_string_literal: true -$LOAD_PATH.unshift(File.expand_path("../lib", __dir__)) +$LOAD_PATH.unshift(File.expand_path("../../lib", __dir__)) require "semian" -require_relative "mock_service" -require_relative "experimental_resource" -require_relative "test_helpers" +require_relative "../mock_service" +require_relative "../experimental_resource" +require_relative "../experiment_helpers" -# Sudden error spike test: 1% -> 20% -> 1% -runner = Semian::Experiments::CircuitBreakerTestRunner.new( - test_name: "Sudden Error Spike Test (Adaptive) - 20% for 20 seconds", +# Sudden error spike experiment: 1% -> 20% -> 1% +runner = Semian::Experiments::CircuitBreakerExperimentRunner.new( + experiment_name: "Sudden Error Spike Experiment (Adaptive) - 20% for 20 seconds", resource_name: "protected_service_sudden_error_spike_20_seconds_adaptive", degradation_phases: [Semian::Experiments::DegradationPhase.new(healthy: true)] * 3 + [Semian::Experiments::DegradationPhase.new(error_rate: 0.20)] + @@ -19,7 +19,7 @@ adaptive_circuit_breaker: true, bulkhead: false, }, - graph_title: "Sudden Error Spike Test (Adaptive) - 20% for 20 seconds", + graph_title: "Sudden Error Spike Experiment (Adaptive) - 20% for 20 seconds", graph_filename: "sudden_error_spike_20_adaptive.png", x_axis_label_interval: 60, ) diff --git a/experiments/test_gradual_increase.rb b/experiments/experiments/experiment_gradual_increase.rb similarity index 74% rename from experiments/test_gradual_increase.rb rename to experiments/experiments/experiment_gradual_increase.rb index 182395012..3b276e997 100644 --- a/experiments/test_gradual_increase.rb +++ b/experiments/experiments/experiment_gradual_increase.rb @@ -1,15 +1,15 @@ # frozen_string_literal: true -$LOAD_PATH.unshift(File.expand_path("../lib", __dir__)) +$LOAD_PATH.unshift(File.expand_path("../../lib", __dir__)) require "semian" -require_relative "mock_service" -require_relative "experimental_resource" -require_relative "test_helpers" +require_relative "../mock_service" +require_relative "../experimental_resource" +require_relative "../experiment_helpers" -# Gradual error increase test: 1% -> 1.5% -> 2% -> ... -> 5% -> 1% -runner = Semian::Experiments::CircuitBreakerTestRunner.new( - test_name: "Gradual Error Increase Test", +# Gradual error increase experiment: 1% -> 1.5% -> 2% -> ... -> 5% -> 1% +runner = Semian::Experiments::CircuitBreakerExperimentRunner.new( + experiment_name: "Gradual Error Increase Experiment", resource_name: "protected_service_gradual", degradation_phases: [ Semian::Experiments::DegradationPhase.new(healthy: true), diff --git a/experiments/test_gradual_increase_adaptive.rb b/experiments/experiments/experiment_gradual_increase_adaptive.rb similarity index 73% rename from experiments/test_gradual_increase_adaptive.rb rename to experiments/experiments/experiment_gradual_increase_adaptive.rb index d31965867..d47c622e8 100644 --- a/experiments/test_gradual_increase_adaptive.rb +++ b/experiments/experiments/experiment_gradual_increase_adaptive.rb @@ -1,15 +1,15 @@ # frozen_string_literal: true -$LOAD_PATH.unshift(File.expand_path("../lib", __dir__)) +$LOAD_PATH.unshift(File.expand_path("../../lib", __dir__)) require "semian" -require_relative "mock_service" -require_relative "experimental_resource" -require_relative "test_helpers" +require_relative "../mock_service" +require_relative "../experimental_resource" +require_relative "../experiment_helpers" -# Gradual error increase test: 1% -> 1.5% -> 2% -> ... -> 5% -> 1% -runner = Semian::Experiments::CircuitBreakerTestRunner.new( - test_name: "Gradual Error Increase Test", +# Gradual error increase experiment: 1% -> 1.5% -> 2% -> ... -> 5% -> 1% +runner = Semian::Experiments::CircuitBreakerExperimentRunner.new( + experiment_name: "Gradual Error Increase Experiment (Adaptive)", resource_name: "protected_service_gradual_adaptive", degradation_phases: [ Semian::Experiments::DegradationPhase.new(healthy: true), diff --git a/experiments/experiments/experiment_lower_bound_windup_adaptive.rb b/experiments/experiments/experiment_lower_bound_windup_adaptive.rb new file mode 100644 index 000000000..6b44cc186 --- /dev/null +++ b/experiments/experiments/experiment_lower_bound_windup_adaptive.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +$LOAD_PATH.unshift(File.expand_path("../../lib", __dir__)) + +require "semian" +require_relative "../mock_service" +require_relative "../experimental_resource" +require_relative "../experiment_helpers" + +# Lower bound windup experiment: demonstrates the response to error spikes +# when integral has accumulated negative values during extended low-error periods. +runner = Semian::Experiments::CircuitBreakerExperimentRunner.new( + experiment_name: "Lower Bound Windup Experiment", + resource_name: "protected_service_windup_demo", + degradation_phases: [Semian::Experiments::DegradationPhase.new(error_rate: 0.01)] + + [Semian::Experiments::DegradationPhase.new(error_rate: 0.60)] * 6 + + [Semian::Experiments::DegradationPhase.new(error_rate: 0.005)] * 120 + + [Semian::Experiments::DegradationPhase.new(error_rate: 0.60)] * 6 + + [Semian::Experiments::DegradationPhase.new(error_rate: 0.01)] * 6, + phase_duration: 30, + semian_config: { + adaptive_circuit_breaker: true, + bulkhead: false, + }, + graph_title: "Adaptive Circuit Breaker: Lower Bound Integral Windup", + graph_filename: "lower_bound_windup.png", + x_axis_label_interval: 60, +) + +runner.run diff --git a/experiments/test_one_of_many_services_degradation.rb b/experiments/experiments/experiment_one_of_many_services_degradation.rb similarity index 71% rename from experiments/test_one_of_many_services_degradation.rb rename to experiments/experiments/experiment_one_of_many_services_degradation.rb index df1860ff3..fc7028ca5 100644 --- a/experiments/test_one_of_many_services_degradation.rb +++ b/experiments/experiments/experiment_one_of_many_services_degradation.rb @@ -1,14 +1,14 @@ # frozen_string_literal: true -$LOAD_PATH.unshift(File.expand_path("../lib", __dir__)) +$LOAD_PATH.unshift(File.expand_path("../../lib", __dir__)) require "semian" -require_relative "mock_service" -require_relative "experimental_resource" -require_relative "test_helpers" +require_relative "../mock_service" +require_relative "../experimental_resource" +require_relative "../experiment_helpers" -runner = Semian::Experiments::CircuitBreakerTestRunner.new( - test_name: "One of Many Services Degradation Test", +runner = Semian::Experiments::CircuitBreakerExperimentRunner.new( + experiment_name: "One of Many Services Degradation Experiment", resource_name: "protected_service", degradation_phases: [Semian::Experiments::DegradationPhase.new(healthy: true)] * 1 + [Semian::Experiments::DegradationPhase.new(latency: 4.95)] * 10 + # Most requests to the target service will timeout diff --git a/experiments/test_one_of_many_services_degradation_adaptive.rb b/experiments/experiments/experiment_one_of_many_services_degradation_adaptive.rb similarity index 70% rename from experiments/test_one_of_many_services_degradation_adaptive.rb rename to experiments/experiments/experiment_one_of_many_services_degradation_adaptive.rb index a3266ec9f..793f7f33d 100644 --- a/experiments/test_one_of_many_services_degradation_adaptive.rb +++ b/experiments/experiments/experiment_one_of_many_services_degradation_adaptive.rb @@ -1,14 +1,14 @@ # frozen_string_literal: true -$LOAD_PATH.unshift(File.expand_path("../lib", __dir__)) +$LOAD_PATH.unshift(File.expand_path("../../lib", __dir__)) require "semian" -require_relative "mock_service" -require_relative "experimental_resource" -require_relative "test_helpers" +require_relative "../mock_service" +require_relative "../experimental_resource" +require_relative "../experiment_helpers" -runner = Semian::Experiments::CircuitBreakerTestRunner.new( - test_name: "One of Many Services Degradation Test", +runner = Semian::Experiments::CircuitBreakerExperimentRunner.new( + experiment_name: "One of Many Services Degradation Experiment", resource_name: "protected_service_adaptive", degradation_phases: [Semian::Experiments::DegradationPhase.new(healthy: true)] * 1 + [Semian::Experiments::DegradationPhase.new(latency: 4.95)] * 10 + # Most requests to the target service will timeout diff --git a/experiments/test_oscillating_errors.rb b/experiments/experiments/experiment_oscillating_errors.rb similarity index 67% rename from experiments/test_oscillating_errors.rb rename to experiments/experiments/experiment_oscillating_errors.rb index effba2f24..52589c938 100644 --- a/experiments/test_oscillating_errors.rb +++ b/experiments/experiments/experiment_oscillating_errors.rb @@ -1,15 +1,15 @@ # frozen_string_literal: true -$LOAD_PATH.unshift(File.expand_path("../lib", __dir__)) +$LOAD_PATH.unshift(File.expand_path("../../lib", __dir__)) require "semian" -require_relative "mock_service" -require_relative "experimental_resource" -require_relative "test_helpers" +require_relative "../mock_service" +require_relative "../experimental_resource" +require_relative "../experiment_helpers" -# Oscillating errors test: 2% <-> 6% errors every 10 seconds -runner = Semian::Experiments::CircuitBreakerTestRunner.new( - test_name: "Oscillating Errors Test", +# Oscillating errors experiment: 2% <-> 6% errors every 10 seconds +runner = Semian::Experiments::CircuitBreakerExperimentRunner.new( + experiment_name: "Oscillating Errors Experiment", resource_name: "protected_service_oscillating", degradation_phases: [Semian::Experiments::DegradationPhase.new(healthy: true)] * 2 + [Semian::Experiments::DegradationPhase.new(error_rate: 0.02), Semian::Experiments::DegradationPhase.new(error_rate: 0.06)] * 9 + diff --git a/experiments/test_oscillating_errors_adaptive.rb b/experiments/experiments/experiment_oscillating_errors_adaptive.rb similarity index 66% rename from experiments/test_oscillating_errors_adaptive.rb rename to experiments/experiments/experiment_oscillating_errors_adaptive.rb index e50f50a1b..81e6721a8 100644 --- a/experiments/test_oscillating_errors_adaptive.rb +++ b/experiments/experiments/experiment_oscillating_errors_adaptive.rb @@ -1,15 +1,15 @@ # frozen_string_literal: true -$LOAD_PATH.unshift(File.expand_path("../lib", __dir__)) +$LOAD_PATH.unshift(File.expand_path("../../lib", __dir__)) require "semian" -require_relative "mock_service" -require_relative "experimental_resource" -require_relative "test_helpers" +require_relative "../mock_service" +require_relative "../experimental_resource" +require_relative "../experiment_helpers" -# Oscillating errors test: 2% <-> 6% errors every 10 seconds -runner = Semian::Experiments::CircuitBreakerTestRunner.new( - test_name: "Oscillating Errors Test", +# Oscillating errors experiment: 2% <-> 6% errors every 10 seconds +runner = Semian::Experiments::CircuitBreakerExperimentRunner.new( + experiment_name: "Oscillating Errors Experiment", resource_name: "protected_service_oscillating_adaptive", degradation_phases: [Semian::Experiments::DegradationPhase.new(healthy: true)] * 2 + [Semian::Experiments::DegradationPhase.new(error_rate: 0.02), Semian::Experiments::DegradationPhase.new(error_rate: 0.06)] * 9 + diff --git a/experiments/test_sudden_error_spikes.rb b/experiments/experiments/experiment_sudden_error_spikes.rb similarity index 79% rename from experiments/test_sudden_error_spikes.rb rename to experiments/experiments/experiment_sudden_error_spikes.rb index ba0682d2e..6b859d1f2 100644 --- a/experiments/test_sudden_error_spikes.rb +++ b/experiments/experiments/experiment_sudden_error_spikes.rb @@ -1,13 +1,13 @@ # frozen_string_literal: true -$LOAD_PATH.unshift(File.expand_path("../lib", __dir__)) +$LOAD_PATH.unshift(File.expand_path("../../lib", __dir__)) require "semian" -require_relative "mock_service" -require_relative "experimental_resource" -require_relative "test_helpers" +require_relative "../mock_service" +require_relative "../experimental_resource" +require_relative "../experiment_helpers" -# Error spikes test: +# Error spikes experiment: # Phase 1: 1% for 60 seconds # Phase 2: 20% for 20 seconds # Phase 3: 1% for 60 seconds @@ -15,8 +15,8 @@ # Phase 5: 1% for 60 seconds # Phase 6: 100% for 20 seconds # Phase 7: 1% for 60 seconds -runner = Semian::Experiments::CircuitBreakerTestRunner.new( - test_name: "Sudden Error Spikes Test (Classic)", +runner = Semian::Experiments::CircuitBreakerExperimentRunner.new( + experiment_name: "Sudden Error Spikes Experiment (Classic)", resource_name: "protected_service_sudden_error_spikes", degradation_phases: [Semian::Experiments::DegradationPhase.new(healthy: true)] * 3 + [Semian::Experiments::DegradationPhase.new(error_rate: 0.20)] + diff --git a/experiments/test_sudden_error_spikes_adaptive.rb b/experiments/experiments/experiment_sudden_error_spikes_adaptive.rb similarity index 78% rename from experiments/test_sudden_error_spikes_adaptive.rb rename to experiments/experiments/experiment_sudden_error_spikes_adaptive.rb index 850004f04..b26efe86a 100644 --- a/experiments/test_sudden_error_spikes_adaptive.rb +++ b/experiments/experiments/experiment_sudden_error_spikes_adaptive.rb @@ -1,13 +1,13 @@ # frozen_string_literal: true -$LOAD_PATH.unshift(File.expand_path("../lib", __dir__)) +$LOAD_PATH.unshift(File.expand_path("../../lib", __dir__)) require "semian" -require_relative "mock_service" -require_relative "experimental_resource" -require_relative "test_helpers" +require_relative "../mock_service" +require_relative "../experimental_resource" +require_relative "../experiment_helpers" -# Error spikes test: +# Error spikes experiment: # Phase 1: 1% for 60 seconds # Phase 2: 20% for 20 seconds # Phase 3: 1% for 60 seconds @@ -15,8 +15,8 @@ # Phase 5: 1% for 60 seconds # Phase 6: 100% for 20 seconds # Phase 7: 1% for 60 seconds -runner = Semian::Experiments::CircuitBreakerTestRunner.new( - test_name: "Sudden Error Spikes Test - adaptive", +runner = Semian::Experiments::CircuitBreakerExperimentRunner.new( + experiment_name: "Sudden Error Spikes Experiment - adaptive", resource_name: "protected_service_sudden_error_spikes_adaptive", degradation_phases: [Semian::Experiments::DegradationPhase.new(healthy: true)] * 3 + [Semian::Experiments::DegradationPhase.new(error_rate: 0.20)] + diff --git a/experiments/test_sustained_load.rb b/experiments/experiments/experiment_sustained_load.rb similarity index 58% rename from experiments/test_sustained_load.rb rename to experiments/experiments/experiment_sustained_load.rb index a79280639..27280e931 100644 --- a/experiments/test_sustained_load.rb +++ b/experiments/experiments/experiment_sustained_load.rb @@ -1,16 +1,16 @@ # frozen_string_literal: true -$LOAD_PATH.unshift(File.expand_path("../lib", __dir__)) +$LOAD_PATH.unshift(File.expand_path("../../lib", __dir__)) require "semian" -require_relative "mock_service" -require_relative "experimental_resource" -require_relative "test_helpers" +require_relative "../mock_service" +require_relative "../experimental_resource" +require_relative "../experiment_helpers" -# Sustained load test: 120s baseline (1%) -> 300s sustained (20%) -> 120s recovery (1%) -runner = Semian::Experiments::CircuitBreakerTestRunner.new( - test_name: "Sustained Load Test", - resource_name: "protected_service", +# Sustained load experiment: 120s baseline (1%) -> 300s sustained (20%) -> 120s recovery (1%) +runner = Semian::Experiments::CircuitBreakerExperimentRunner.new( + experiment_name: "Sustained Load Experiment (Classic)", + resource_name: "protected_service_sustained_load", degradation_phases: [Semian::Experiments::DegradationPhase.new(healthy: true)] * 4 + [Semian::Experiments::DegradationPhase.new(error_rate: 0.20)] * 10 + [Semian::Experiments::DegradationPhase.new(healthy: true)] * 4, diff --git a/experiments/test_sustained_load_adaptive.rb b/experiments/experiments/experiment_sustained_load_adaptive.rb similarity index 55% rename from experiments/test_sustained_load_adaptive.rb rename to experiments/experiments/experiment_sustained_load_adaptive.rb index 15633586f..bcb1dcaa3 100644 --- a/experiments/test_sustained_load_adaptive.rb +++ b/experiments/experiments/experiment_sustained_load_adaptive.rb @@ -1,16 +1,16 @@ # frozen_string_literal: true -$LOAD_PATH.unshift(File.expand_path("../lib", __dir__)) +$LOAD_PATH.unshift(File.expand_path("../../lib", __dir__)) require "semian" -require_relative "mock_service" -require_relative "experimental_resource" -require_relative "test_helpers" +require_relative "../mock_service" +require_relative "../experimental_resource" +require_relative "../experiment_helpers" -# Sustained load test: 120s baseline (1%) -> 300s sustained (20%) -> 120s recovery (1%) -runner = Semian::Experiments::CircuitBreakerTestRunner.new( - test_name: "Sustained Load Test", - resource_name: "protected_service_adaptive", +# Sustained load experiment: 120s baseline (1%) -> 300s sustained (20%) -> 120s recovery (1%) +runner = Semian::Experiments::CircuitBreakerExperimentRunner.new( + experiment_name: "Sustained Load Experiment (Adaptive)", + resource_name: "protected_service_sustained_load_adaptive", degradation_phases: [Semian::Experiments::DegradationPhase.new(healthy: true)] * 4 + [Semian::Experiments::DegradationPhase.new(error_rate: 0.20)] * 10 + [Semian::Experiments::DegradationPhase.new(healthy: true)] * 4, diff --git a/experiments/gradual_increase.png b/experiments/gradual_increase.png deleted file mode 100644 index 7ee4c29f7..000000000 Binary files a/experiments/gradual_increase.png and /dev/null differ diff --git a/experiments/gradual_increase_adaptive.png b/experiments/gradual_increase_adaptive.png deleted file mode 100644 index b66bb762c..000000000 Binary files a/experiments/gradual_increase_adaptive.png and /dev/null differ diff --git a/experiments/one_of_many_services_latency_degradation.png b/experiments/one_of_many_services_latency_degradation.png deleted file mode 100644 index 601fcc696..000000000 Binary files a/experiments/one_of_many_services_latency_degradation.png and /dev/null differ diff --git a/experiments/one_of_many_services_latency_degradation_adaptive.png b/experiments/one_of_many_services_latency_degradation_adaptive.png deleted file mode 100644 index 145a27edf..000000000 Binary files a/experiments/one_of_many_services_latency_degradation_adaptive.png and /dev/null differ diff --git a/experiments/oscillating_errors.png b/experiments/oscillating_errors.png deleted file mode 100644 index 9cf5b5e60..000000000 Binary files a/experiments/oscillating_errors.png and /dev/null differ diff --git a/experiments/oscillating_errors_adaptive.png b/experiments/oscillating_errors_adaptive.png deleted file mode 100644 index 25164cd5a..000000000 Binary files a/experiments/oscillating_errors_adaptive.png and /dev/null differ diff --git a/experiments/results/duration_graphs/duration-gradual_increase.png b/experiments/results/duration_graphs/duration-gradual_increase.png new file mode 100644 index 000000000..b317bad44 Binary files /dev/null and b/experiments/results/duration_graphs/duration-gradual_increase.png differ diff --git a/experiments/results/duration_graphs/duration-gradual_increase_adaptive.png b/experiments/results/duration_graphs/duration-gradual_increase_adaptive.png new file mode 100644 index 000000000..48bef1c24 Binary files /dev/null and b/experiments/results/duration_graphs/duration-gradual_increase_adaptive.png differ diff --git a/experiments/duration-lower_bound_windup.png b/experiments/results/duration_graphs/duration-lower_bound_windup.png similarity index 100% rename from experiments/duration-lower_bound_windup.png rename to experiments/results/duration_graphs/duration-lower_bound_windup.png diff --git a/experiments/results/duration_graphs/duration-one_of_many_services_latency_degradation.png b/experiments/results/duration_graphs/duration-one_of_many_services_latency_degradation.png new file mode 100644 index 000000000..b8db9b2f0 Binary files /dev/null and b/experiments/results/duration_graphs/duration-one_of_many_services_latency_degradation.png differ diff --git a/experiments/results/duration_graphs/duration-one_of_many_services_latency_degradation_adaptive.png b/experiments/results/duration_graphs/duration-one_of_many_services_latency_degradation_adaptive.png new file mode 100644 index 000000000..185eb6ca0 Binary files /dev/null and b/experiments/results/duration_graphs/duration-one_of_many_services_latency_degradation_adaptive.png differ diff --git a/experiments/results/duration_graphs/duration-oscillating_errors.png b/experiments/results/duration_graphs/duration-oscillating_errors.png new file mode 100644 index 000000000..47694bdb8 Binary files /dev/null and b/experiments/results/duration_graphs/duration-oscillating_errors.png differ diff --git a/experiments/results/duration_graphs/duration-oscillating_errors_adaptive.png b/experiments/results/duration_graphs/duration-oscillating_errors_adaptive.png new file mode 100644 index 000000000..82dde6789 Binary files /dev/null and b/experiments/results/duration_graphs/duration-oscillating_errors_adaptive.png differ diff --git a/experiments/results/duration_graphs/duration-sudden_error_spike_100.png b/experiments/results/duration_graphs/duration-sudden_error_spike_100.png new file mode 100644 index 000000000..56aeefd1c Binary files /dev/null and b/experiments/results/duration_graphs/duration-sudden_error_spike_100.png differ diff --git a/experiments/results/duration_graphs/duration-sudden_error_spike_100_adaptive.png b/experiments/results/duration_graphs/duration-sudden_error_spike_100_adaptive.png new file mode 100644 index 000000000..64c674701 Binary files /dev/null and b/experiments/results/duration_graphs/duration-sudden_error_spike_100_adaptive.png differ diff --git a/experiments/results/duration_graphs/duration-sudden_error_spike_20.png b/experiments/results/duration_graphs/duration-sudden_error_spike_20.png new file mode 100644 index 000000000..8ecceeae5 Binary files /dev/null and b/experiments/results/duration_graphs/duration-sudden_error_spike_20.png differ diff --git a/experiments/results/duration_graphs/duration-sudden_error_spike_20_adaptive.png b/experiments/results/duration_graphs/duration-sudden_error_spike_20_adaptive.png new file mode 100644 index 000000000..38e8443f2 Binary files /dev/null and b/experiments/results/duration_graphs/duration-sudden_error_spike_20_adaptive.png differ diff --git a/experiments/results/duration_graphs/duration-sudden_error_spikes.png b/experiments/results/duration_graphs/duration-sudden_error_spikes.png new file mode 100644 index 000000000..ea45f41cf Binary files /dev/null and b/experiments/results/duration_graphs/duration-sudden_error_spikes.png differ diff --git a/experiments/results/duration_graphs/duration-sudden_error_spikes_adaptive.png b/experiments/results/duration_graphs/duration-sudden_error_spikes_adaptive.png new file mode 100644 index 000000000..96df8ff2d Binary files /dev/null and b/experiments/results/duration_graphs/duration-sudden_error_spikes_adaptive.png differ diff --git a/experiments/results/duration_graphs/duration-sustained_load.png b/experiments/results/duration_graphs/duration-sustained_load.png new file mode 100644 index 000000000..8400e2e69 Binary files /dev/null and b/experiments/results/duration_graphs/duration-sustained_load.png differ diff --git a/experiments/results/duration_graphs/duration-sustained_load_adaptive.png b/experiments/results/duration_graphs/duration-sustained_load_adaptive.png new file mode 100644 index 000000000..4920907e7 Binary files /dev/null and b/experiments/results/duration_graphs/duration-sustained_load_adaptive.png differ diff --git a/experiments/results/main_graphs/gradual_increase.png b/experiments/results/main_graphs/gradual_increase.png new file mode 100644 index 000000000..9f7efc08f Binary files /dev/null and b/experiments/results/main_graphs/gradual_increase.png differ diff --git a/experiments/results/main_graphs/gradual_increase_adaptive.png b/experiments/results/main_graphs/gradual_increase_adaptive.png new file mode 100644 index 000000000..9ebecf75f Binary files /dev/null and b/experiments/results/main_graphs/gradual_increase_adaptive.png differ diff --git a/experiments/lower_bound_windup.png b/experiments/results/main_graphs/lower_bound_windup.png similarity index 100% rename from experiments/lower_bound_windup.png rename to experiments/results/main_graphs/lower_bound_windup.png diff --git a/experiments/results/main_graphs/one_of_many_services_latency_degradation.png b/experiments/results/main_graphs/one_of_many_services_latency_degradation.png new file mode 100644 index 000000000..12f657091 Binary files /dev/null and b/experiments/results/main_graphs/one_of_many_services_latency_degradation.png differ diff --git a/experiments/results/main_graphs/one_of_many_services_latency_degradation_adaptive.png b/experiments/results/main_graphs/one_of_many_services_latency_degradation_adaptive.png new file mode 100644 index 000000000..5654f9a8d Binary files /dev/null and b/experiments/results/main_graphs/one_of_many_services_latency_degradation_adaptive.png differ diff --git a/experiments/results/main_graphs/oscillating_errors.png b/experiments/results/main_graphs/oscillating_errors.png new file mode 100644 index 000000000..7cc3f25ca Binary files /dev/null and b/experiments/results/main_graphs/oscillating_errors.png differ diff --git a/experiments/results/main_graphs/oscillating_errors_adaptive.png b/experiments/results/main_graphs/oscillating_errors_adaptive.png new file mode 100644 index 000000000..0c4433f51 Binary files /dev/null and b/experiments/results/main_graphs/oscillating_errors_adaptive.png differ diff --git a/experiments/results/main_graphs/sudden_error_spike_100.png b/experiments/results/main_graphs/sudden_error_spike_100.png new file mode 100644 index 000000000..ac1480711 Binary files /dev/null and b/experiments/results/main_graphs/sudden_error_spike_100.png differ diff --git a/experiments/results/main_graphs/sudden_error_spike_100_adaptive.png b/experiments/results/main_graphs/sudden_error_spike_100_adaptive.png new file mode 100644 index 000000000..5ad8e6a08 Binary files /dev/null and b/experiments/results/main_graphs/sudden_error_spike_100_adaptive.png differ diff --git a/experiments/results/main_graphs/sudden_error_spike_20.png b/experiments/results/main_graphs/sudden_error_spike_20.png new file mode 100644 index 000000000..cc47edfd6 Binary files /dev/null and b/experiments/results/main_graphs/sudden_error_spike_20.png differ diff --git a/experiments/results/main_graphs/sudden_error_spike_20_adaptive.png b/experiments/results/main_graphs/sudden_error_spike_20_adaptive.png new file mode 100644 index 000000000..9a9b7f897 Binary files /dev/null and b/experiments/results/main_graphs/sudden_error_spike_20_adaptive.png differ diff --git a/experiments/results/main_graphs/sudden_error_spikes.png b/experiments/results/main_graphs/sudden_error_spikes.png new file mode 100644 index 000000000..e5c5e7f69 Binary files /dev/null and b/experiments/results/main_graphs/sudden_error_spikes.png differ diff --git a/experiments/results/main_graphs/sudden_error_spikes_adaptive.png b/experiments/results/main_graphs/sudden_error_spikes_adaptive.png new file mode 100644 index 000000000..f19f078d9 Binary files /dev/null and b/experiments/results/main_graphs/sudden_error_spikes_adaptive.png differ diff --git a/experiments/results/main_graphs/sustained_load.png b/experiments/results/main_graphs/sustained_load.png new file mode 100644 index 000000000..54038971b Binary files /dev/null and b/experiments/results/main_graphs/sustained_load.png differ diff --git a/experiments/results/main_graphs/sustained_load_adaptive.png b/experiments/results/main_graphs/sustained_load_adaptive.png new file mode 100644 index 000000000..9474e3ffa Binary files /dev/null and b/experiments/results/main_graphs/sustained_load_adaptive.png differ diff --git a/experiments/results/throughput_graphs/throughput-gradual_increase.png b/experiments/results/throughput_graphs/throughput-gradual_increase.png new file mode 100644 index 000000000..3ef895b16 Binary files /dev/null and b/experiments/results/throughput_graphs/throughput-gradual_increase.png differ diff --git a/experiments/results/throughput_graphs/throughput-gradual_increase_adaptive.png b/experiments/results/throughput_graphs/throughput-gradual_increase_adaptive.png new file mode 100644 index 000000000..63b6beee8 Binary files /dev/null and b/experiments/results/throughput_graphs/throughput-gradual_increase_adaptive.png differ diff --git a/experiments/results/throughput_graphs/throughput-one_of_many_services_latency_degradation.png b/experiments/results/throughput_graphs/throughput-one_of_many_services_latency_degradation.png new file mode 100644 index 000000000..b47b6b8a2 Binary files /dev/null and b/experiments/results/throughput_graphs/throughput-one_of_many_services_latency_degradation.png differ diff --git a/experiments/results/throughput_graphs/throughput-one_of_many_services_latency_degradation_adaptive.png b/experiments/results/throughput_graphs/throughput-one_of_many_services_latency_degradation_adaptive.png new file mode 100644 index 000000000..14673f5cc Binary files /dev/null and b/experiments/results/throughput_graphs/throughput-one_of_many_services_latency_degradation_adaptive.png differ diff --git a/experiments/results/throughput_graphs/throughput-oscillating_errors.png b/experiments/results/throughput_graphs/throughput-oscillating_errors.png new file mode 100644 index 000000000..308bd0f04 Binary files /dev/null and b/experiments/results/throughput_graphs/throughput-oscillating_errors.png differ diff --git a/experiments/results/throughput_graphs/throughput-oscillating_errors_adaptive.png b/experiments/results/throughput_graphs/throughput-oscillating_errors_adaptive.png new file mode 100644 index 000000000..fc2249093 Binary files /dev/null and b/experiments/results/throughput_graphs/throughput-oscillating_errors_adaptive.png differ diff --git a/experiments/results/throughput_graphs/throughput-sudden_error_spike_100.png b/experiments/results/throughput_graphs/throughput-sudden_error_spike_100.png new file mode 100644 index 000000000..8ed522998 Binary files /dev/null and b/experiments/results/throughput_graphs/throughput-sudden_error_spike_100.png differ diff --git a/experiments/results/throughput_graphs/throughput-sudden_error_spike_100_adaptive.png b/experiments/results/throughput_graphs/throughput-sudden_error_spike_100_adaptive.png new file mode 100644 index 000000000..6744f0549 Binary files /dev/null and b/experiments/results/throughput_graphs/throughput-sudden_error_spike_100_adaptive.png differ diff --git a/experiments/results/throughput_graphs/throughput-sudden_error_spike_20.png b/experiments/results/throughput_graphs/throughput-sudden_error_spike_20.png new file mode 100644 index 000000000..7d64c9b88 Binary files /dev/null and b/experiments/results/throughput_graphs/throughput-sudden_error_spike_20.png differ diff --git a/experiments/results/throughput_graphs/throughput-sudden_error_spike_20_adaptive.png b/experiments/results/throughput_graphs/throughput-sudden_error_spike_20_adaptive.png new file mode 100644 index 000000000..e567b2cd1 Binary files /dev/null and b/experiments/results/throughput_graphs/throughput-sudden_error_spike_20_adaptive.png differ diff --git a/experiments/results/throughput_graphs/throughput-sudden_error_spikes.png b/experiments/results/throughput_graphs/throughput-sudden_error_spikes.png new file mode 100644 index 000000000..1488f23c4 Binary files /dev/null and b/experiments/results/throughput_graphs/throughput-sudden_error_spikes.png differ diff --git a/experiments/results/throughput_graphs/throughput-sudden_error_spikes_adaptive.png b/experiments/results/throughput_graphs/throughput-sudden_error_spikes_adaptive.png new file mode 100644 index 000000000..796b8e0c2 Binary files /dev/null and b/experiments/results/throughput_graphs/throughput-sudden_error_spikes_adaptive.png differ diff --git a/experiments/results/throughput_graphs/throughput-sustained_load.png b/experiments/results/throughput_graphs/throughput-sustained_load.png new file mode 100644 index 000000000..189a12a0a Binary files /dev/null and b/experiments/results/throughput_graphs/throughput-sustained_load.png differ diff --git a/experiments/results/throughput_graphs/throughput-sustained_load_adaptive.png b/experiments/results/throughput_graphs/throughput-sustained_load_adaptive.png new file mode 100644 index 000000000..8cb13c193 Binary files /dev/null and b/experiments/results/throughput_graphs/throughput-sustained_load_adaptive.png differ diff --git a/experiments/run_all_experiments.rb b/experiments/run_all_experiments.rb new file mode 100755 index 000000000..5ec2d379b --- /dev/null +++ b/experiments/run_all_experiments.rb @@ -0,0 +1,91 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require "thread" + +# Get all experiment files (excluding the windup experiment) +# TODO: Include lower bound windup experiment once we have a way to make it run in a reasonable time. +experiment_files = Dir.glob("experiments/*.rb").reject { |file| file.include?("experiment_lower_bound_windup_adaptive.rb") }.sort + +puts "Found #{experiment_files.length} experiment files to run:" +experiment_files.each { |file| puts " - #{file}" } +puts "\nRunning all experiments in parallel..." +puts "=" * 60 + +# Track results +results = {} +mutex = Mutex.new + +# Create threads for each experiment file +threads = experiment_files.map do |experiment_file| + Thread.new do + start_time = Time.now + + # Capture output and error + output = nil + error = nil + exit_status = nil + + begin + # Run the experiment file and capture output + output = %x(bundle exec ruby #{experiment_file} 2>&1) + exit_status = $?.exitstatus + rescue => e + error = e.message + exit_status = 1 + end + + end_time = Time.now + duration = end_time - start_time + + # Thread-safe result storage + mutex.synchronize do + results[experiment_file] = { + output: output, + error: error, + exit_status: exit_status, + duration: duration, + } + + status = exit_status == 0 ? "✅ SUCCESS" : "❌ FAILED" + puts "[#{Time.now.strftime("%H:%M:%S")}] #{status} #{experiment_file} (#{duration.round(2)}s)" + end + end +end + +# Wait for all threads to complete +threads.each(&:join) + +puts "\n" + "=" * 60 +puts "All experiments completed!" +puts "=" * 60 + +# Summary +total_duration = results.values.map { |r| r[:duration] }.sum +successful = results.select { |_, r| r[:exit_status] == 0 } +failed = results.select { |_, r| r[:exit_status] != 0 } + +puts "\nSUMMARY:" +puts " Total files: #{results.length}" +puts " Successful: #{successful.length}" +puts " Failed: #{failed.length}" +puts " Total execution time: #{total_duration.round(2)}s" + +# Show failed experiments with details +if failed.any? + puts "\nFAILED EXPERIMENTS:" + failed.each do |file, result| + puts "\n#{file}:" + puts " Exit status: #{result[:exit_status]}" + puts " Duration: #{result[:duration].round(2)}s" + if result[:error] + puts " Error: #{result[:error]}" + end + if result[:output] && !result[:output].strip.empty? + puts " Output:" + result[:output].split("\n").each { |line| puts " #{line}" } + end + end +end + +exit failed.any? ? 1 : 0 diff --git a/experiments/sudden_error_spike_100.png b/experiments/sudden_error_spike_100.png deleted file mode 100644 index 04b8c8afb..000000000 Binary files a/experiments/sudden_error_spike_100.png and /dev/null differ diff --git a/experiments/sudden_error_spike_100_adaptive.png b/experiments/sudden_error_spike_100_adaptive.png deleted file mode 100644 index 35683822c..000000000 Binary files a/experiments/sudden_error_spike_100_adaptive.png and /dev/null differ diff --git a/experiments/sudden_error_spike_20.png b/experiments/sudden_error_spike_20.png deleted file mode 100644 index 15214adcf..000000000 Binary files a/experiments/sudden_error_spike_20.png and /dev/null differ diff --git a/experiments/sudden_error_spike_20_adaptive.png b/experiments/sudden_error_spike_20_adaptive.png deleted file mode 100644 index 2f8306034..000000000 Binary files a/experiments/sudden_error_spike_20_adaptive.png and /dev/null differ diff --git a/experiments/sudden_error_spikes_adaptive.png b/experiments/sudden_error_spikes_adaptive.png deleted file mode 100644 index 26f0aea2f..000000000 Binary files a/experiments/sudden_error_spikes_adaptive.png and /dev/null differ diff --git a/experiments/sustained_load.png b/experiments/sustained_load.png deleted file mode 100644 index 0661d36e3..000000000 Binary files a/experiments/sustained_load.png and /dev/null differ diff --git a/experiments/sustained_load_adaptive.png b/experiments/sustained_load_adaptive.png deleted file mode 100644 index 99dcad6cd..000000000 Binary files a/experiments/sustained_load_adaptive.png and /dev/null differ diff --git a/experiments/test_lower_bound_windup_adaptive.rb b/experiments/test_lower_bound_windup_adaptive.rb deleted file mode 100644 index eeebad28c..000000000 --- a/experiments/test_lower_bound_windup_adaptive.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true - -$LOAD_PATH.unshift(File.expand_path("../lib", __dir__)) - -require "semian" -require_relative "mock_service" -require_relative "experimental_resource" -require_relative "test_helpers" - -# Lower bound windup test: demonstrates the response to error spikes -# when integral has accumulated negative values during extended low-error periods. -runner = Semian::Experiments::CircuitBreakerTestRunner.new( - test_name: "Lower Bound Windup Test", - resource_name: "protected_service_windup_demo", - error_phases: [0.01] + [0.60] * 6 + [0.005] * 120 + [0.60] * 6 + [0.01] * 6, - phase_duration: 30, - semian_config: { - adaptive_circuit_breaker: true, - bulkhead: false, - }, - graph_title: "Adaptive Circuit Breaker: Lower Bound Integral Windup", - graph_filename: "lower_bound_windup.png", - x_axis_label_interval: 60, -) - -runner.run diff --git a/experiments/throughput-one_of_many_services_latency_degradation.png b/experiments/throughput-one_of_many_services_latency_degradation.png deleted file mode 100644 index 6a04f3aca..000000000 Binary files a/experiments/throughput-one_of_many_services_latency_degradation.png and /dev/null differ diff --git a/experiments/throughput-one_of_many_services_latency_degradation_adaptive.png b/experiments/throughput-one_of_many_services_latency_degradation_adaptive.png deleted file mode 100644 index b16eb7cb7..000000000 Binary files a/experiments/throughput-one_of_many_services_latency_degradation_adaptive.png and /dev/null differ