diff --git a/lib/agent_forge/execution_stats.ex b/lib/agent_forge/execution_stats.ex new file mode 100644 index 0000000..50b538b --- /dev/null +++ b/lib/agent_forge/execution_stats.ex @@ -0,0 +1,133 @@ +defmodule AgentForge.ExecutionStats do + @moduledoc """ + Provides functionality for collecting and analyzing execution statistics of flows. + Tracks metrics such as execution time, steps taken, and signal patterns. + """ + + @type t :: %__MODULE__{ + start_time: integer(), + steps: non_neg_integer(), + signal_types: %{atom() => non_neg_integer()}, + handler_calls: %{atom() => non_neg_integer()}, + max_state_size: non_neg_integer(), + complete: boolean(), + elapsed_ms: integer() | nil, + result: any() + } + + defstruct start_time: nil, + steps: 0, + signal_types: %{}, + handler_calls: %{}, + max_state_size: 0, + complete: false, + elapsed_ms: nil, + result: nil + + @doc """ + Creates a new execution stats struct with initial values. + + ## Examples + + iex> stats = AgentForge.ExecutionStats.new() + iex> is_integer(stats.start_time) and stats.steps == 0 + true + """ + def new do + %__MODULE__{ + start_time: System.monotonic_time(:millisecond) + } + end + + @doc """ + Records a step in the execution process, updating relevant statistics. + + ## Parameters + + * `stats` - Current execution stats struct + * `handler_info` - Information about the handler being executed + * `signal` - The signal being processed + * `state` - Current state of the flow + + ## Examples + + iex> stats = AgentForge.ExecutionStats.new() + iex> signal = %{type: :test, data: "data"} + iex> updated = AgentForge.ExecutionStats.record_step(stats, :test_handler, signal, %{}) + iex> updated.steps == 1 and updated.signal_types == %{test: 1} + true + """ + def record_step(stats, handler_info, signal, state) do + state_size = get_state_size(state) + + %{ + stats + | steps: stats.steps + 1, + signal_types: increment_counter(stats.signal_types, signal.type), + handler_calls: increment_counter(stats.handler_calls, handler_info), + max_state_size: max(stats.max_state_size, state_size) + } + end + + @doc """ + Finalizes the execution stats with the result and calculates elapsed time. + + ## Parameters + + * `stats` - Current execution stats struct + * `result` - The final result of the flow execution + + ## Examples + + iex> stats = AgentForge.ExecutionStats.new() + iex> final = AgentForge.ExecutionStats.finalize(stats, {:ok, "success"}) + iex> final.complete and is_integer(final.elapsed_ms) and final.result == {:ok, "success"} + true + """ + def finalize(stats, result) do + %{ + stats + | complete: true, + elapsed_ms: System.monotonic_time(:millisecond) - stats.start_time, + result: result + } + end + + @doc """ + Formats the execution stats into a human-readable report. + + ## Examples + + iex> stats = AgentForge.ExecutionStats.new() + iex> stats = AgentForge.ExecutionStats.finalize(stats, {:ok, "success"}) + iex> report = AgentForge.ExecutionStats.format_report(stats) + iex> String.contains?(report, "Total Steps: 0") and String.contains?(report, "Result: {:ok, \\"success\\"}") + true + """ + def format_report(stats) do + """ + Execution Statistics: + - Total Steps: #{stats.steps} + - Elapsed Time: #{stats.elapsed_ms}ms + - Signal Types: #{format_counters(stats.signal_types)} + - Handler Calls: #{format_counters(stats.handler_calls)} + - Max State Size: #{stats.max_state_size} entries + - Result: #{inspect(stats.result)} + """ + end + + # Private Functions + + defp increment_counter(counters, key) do + Map.update(counters, key, 1, &(&1 + 1)) + end + + defp get_state_size(state) when is_map(state), do: map_size(state) + defp get_state_size(_), do: 0 + + defp format_counters(counters) do + counters + |> Enum.map(fn {key, count} -> "#{key}: #{count}" end) + |> Enum.join(", ") + end +end diff --git a/lib/agent_forge/flow.ex b/lib/agent_forge/flow.ex index 0ced868..0e8d310 100644 --- a/lib/agent_forge/flow.ex +++ b/lib/agent_forge/flow.ex @@ -2,9 +2,14 @@ defmodule AgentForge.Flow do @moduledoc """ Provides functions for processing signals through a chain of handlers. Each handler is a function that takes a signal and state, and returns a tuple with result and new state. + Automatically collects execution statistics for monitoring and debugging. """ alias AgentForge.Signal + alias AgentForge.ExecutionStats + + # Store last execution stats in module attribute + @last_execution_stats_key :"$agent_forge_last_execution_stats" @doc """ Processes a signal through a list of handlers. @@ -61,27 +66,34 @@ defmodule AgentForge.Flow do # Private functions defp process_handlers(handlers, signal, state) do - Enum.reduce_while(handlers, {:ok, signal, state}, fn handler, - {:ok, current_signal, current_state} -> + stats = ExecutionStats.new() + + Enum.reduce_while(handlers, {:ok, signal, state, stats}, fn handler, + {:ok, current_signal, + current_state, current_stats} -> + # Record step before processing + updated_stats = + ExecutionStats.record_step(current_stats, handler, current_signal, current_state) + case process_handler(handler, current_signal, current_state) do {{:emit, new_signal}, new_state} -> - {:cont, {:ok, new_signal, new_state}} + {:cont, {:ok, new_signal, new_state, updated_stats}} {{:emit_many, signals}, new_state} when is_list(signals) -> # When multiple signals are emitted, use the last one for continuation - {:cont, {:ok, List.last(signals), new_state}} + {:cont, {:ok, List.last(signals), new_state, updated_stats}} {:skip, new_state} -> - {:halt, {:ok, nil, new_state}} + {:halt, {:ok, nil, new_state, updated_stats}} {:halt, data} -> - {:halt, {:ok, data, state}} + {:halt, {:ok, data, state, updated_stats}} {{:halt, data}, _state} -> - {:halt, {:ok, data, state}} + {:halt, {:ok, data, state, updated_stats}} {{:error, reason}, new_state} -> - {:halt, {:error, reason, new_state}} + {:halt, {:error, reason, new_state, updated_stats}} {other, _} -> raise "Invalid handler result: #{inspect(other)}" @@ -93,6 +105,23 @@ defmodule AgentForge.Flow do end # Handle the final result - defp handle_result({:ok, signal, state}), do: {:ok, signal, state} - defp handle_result({:error, reason, _state}), do: {:error, reason} + defp handle_result({:ok, signal, state, stats}) do + final_stats = ExecutionStats.finalize(stats, {:ok, signal}) + Process.put(@last_execution_stats_key, final_stats) + {:ok, signal, state} + end + + defp handle_result({:error, reason, _state, stats}) do + final_stats = ExecutionStats.finalize(stats, {:error, reason}) + Process.put(@last_execution_stats_key, final_stats) + {:error, reason} + end + + @doc """ + Returns statistics from the last flow execution. + Returns nil if no flow has been executed yet. + """ + def get_last_execution_stats do + Process.get(@last_execution_stats_key) + end end diff --git a/test/agent_forge/execution_stats_test.exs b/test/agent_forge/execution_stats_test.exs new file mode 100644 index 0000000..b157977 --- /dev/null +++ b/test/agent_forge/execution_stats_test.exs @@ -0,0 +1,99 @@ +defmodule AgentForge.ExecutionStatsTest do + use ExUnit.Case + doctest AgentForge.ExecutionStats + + alias AgentForge.ExecutionStats + alias AgentForge.Signal + + describe "new/0" do + test "creates new stats with initial values" do + stats = ExecutionStats.new() + assert stats.steps == 0 + assert stats.signal_types == %{} + assert stats.handler_calls == %{} + assert stats.max_state_size == 0 + assert stats.complete == false + assert stats.elapsed_ms == nil + assert stats.result == nil + assert is_integer(stats.start_time) + end + end + + describe "record_step/4" do + test "increments steps and tracks signal types" do + stats = ExecutionStats.new() + signal = Signal.new(:test_signal, "data") + state = %{key: "value"} + + updated_stats = ExecutionStats.record_step(stats, :test_handler, signal, state) + + assert updated_stats.steps == 1 + assert updated_stats.signal_types == %{test_signal: 1} + assert updated_stats.handler_calls == %{test_handler: 1} + assert updated_stats.max_state_size == 1 + end + + test "tracks multiple signal types and handlers" do + stats = ExecutionStats.new() + signal1 = Signal.new(:type_a, "data1") + signal2 = Signal.new(:type_b, "data2") + signal3 = Signal.new(:type_a, "data3") + state = %{key1: "value1", key2: "value2"} + + stats = ExecutionStats.record_step(stats, :handler1, signal1, %{}) + stats = ExecutionStats.record_step(stats, :handler2, signal2, state) + stats = ExecutionStats.record_step(stats, :handler1, signal3, state) + + assert stats.steps == 3 + assert stats.signal_types == %{type_a: 2, type_b: 1} + assert stats.handler_calls == %{handler1: 2, handler2: 1} + assert stats.max_state_size == 2 + end + + test "handles non-map state" do + stats = ExecutionStats.new() + signal = Signal.new(:test, "data") + + updated_stats = ExecutionStats.record_step(stats, :handler, signal, nil) + + assert updated_stats.steps == 1 + assert updated_stats.max_state_size == 0 + end + end + + describe "finalize/2" do + test "completes stats with result and elapsed time" do + stats = ExecutionStats.new() + result = {:ok, "success"} + + # Ensure some time passes + :timer.sleep(1) + stats = ExecutionStats.finalize(stats, result) + + assert stats.complete == true + assert stats.result == result + assert stats.elapsed_ms > 0 + end + end + + describe "format_report/1" do + test "generates readable report" do + stats = ExecutionStats.new() + signal = Signal.new(:test_signal, "data") + + stats = + stats + |> ExecutionStats.record_step(:handler, signal, %{a: 1}) + |> ExecutionStats.finalize({:ok, "done"}) + + report = ExecutionStats.format_report(stats) + + assert is_binary(report) + assert report =~ "Total Steps: 1" + assert report =~ "Signal Types: test_signal: 1" + assert report =~ "Handler Calls: handler: 1" + assert report =~ "Max State Size: 1" + assert report =~ ~s(Result: {:ok, "done"}) + end + end +end diff --git a/test/agent_forge/flow_stats_test.exs b/test/agent_forge/flow_stats_test.exs new file mode 100644 index 0000000..4a73caa --- /dev/null +++ b/test/agent_forge/flow_stats_test.exs @@ -0,0 +1,117 @@ +defmodule AgentForge.FlowStatsTest do + use ExUnit.Case + + alias AgentForge.Flow + alias AgentForge.Signal + alias AgentForge.ExecutionStats + + setup do + # Clear any previous execution stats before each test + Process.put(:"$agent_forge_last_execution_stats", nil) + :ok + end + + describe "flow execution statistics" do + test "collects basic execution stats" do + signal = Signal.new(:test, "data") + handler = fn sig, state -> {{:emit, sig}, state} end + + assert {:ok, ^signal, %{}} = Flow.process([handler], signal, %{}) + + stats = Flow.get_last_execution_stats() + assert %ExecutionStats{} = stats + assert stats.steps == 1 + assert stats.signal_types == %{test: 1} + assert stats.complete == true + assert is_integer(stats.elapsed_ms) and stats.elapsed_ms >= 0 + assert stats.result == {:ok, signal} + end + + test "tracks multiple handlers" do + signal = Signal.new(:initial, "data") + + handlers = [ + fn sig, state -> {{:emit, Signal.new(:step1, sig.data)}, state} end, + fn sig, state -> {{:emit, Signal.new(:step2, sig.data)}, state} end, + fn sig, state -> {{:emit, Signal.new(:final, sig.data)}, state} end + ] + + Flow.process(handlers, signal, %{}) + stats = Flow.get_last_execution_stats() + + assert stats.steps == 3 + assert stats.signal_types == %{initial: 1, step1: 1, step2: 1} + # Handler calls are tracked by function reference, so just verify the count + assert map_size(stats.handler_calls) == 3 + assert Enum.all?(Map.values(stats.handler_calls), &(&1 == 1)) + end + + test "handles early termination with skip" do + signal = Signal.new(:test, "data") + + handlers = [ + fn _sig, state -> {:skip, state} end, + fn _sig, state -> {{:emit, Signal.new(:never_reached, "data")}, state} end + ] + + Flow.process(handlers, signal, %{}) + stats = Flow.get_last_execution_stats() + + assert stats.steps == 1 + assert stats.signal_types == %{test: 1} + assert stats.result == {:ok, nil} + end + + test "handles errors in flow" do + signal = Signal.new(:test, "data") + + handlers = [ + fn _sig, state -> {{:error, "test error"}, state} end + ] + + assert {:error, "test error"} = Flow.process(handlers, signal, %{}) + stats = Flow.get_last_execution_stats() + + assert stats.steps == 1 + assert stats.signal_types == %{test: 1} + assert stats.result == {:error, "test error"} + end + + test "tracks state size changes" do + signal = Signal.new(:test, "data") + + handlers = [ + fn sig, state -> {{:emit, sig}, Map.put(state, :a, 1)} end, + fn sig, state -> {{:emit, sig}, Map.put(state, :b, 2)} end, + fn sig, state -> {{:emit, sig}, Map.delete(state, :a)} end + ] + + Flow.process(handlers, signal, %{}) + stats = Flow.get_last_execution_stats() + + assert stats.max_state_size == 2 + end + + test "handles emit_many signals" do + signal = Signal.new(:test, "data") + + handlers = [ + fn _sig, state -> + signals = [ + Signal.new(:first, "data1"), + Signal.new(:second, "data2") + ] + + {{:emit_many, signals}, state} + end + ] + + Flow.process(handlers, signal, %{}) + stats = Flow.get_last_execution_stats() + + assert stats.steps == 1 + assert stats.signal_types == %{test: 1} + assert stats.complete == true + end + end +end