From bbaaaf052e0454539d294a9b4c1c68f36d6f25fd Mon Sep 17 00:00:00 2001 From: JediLuke <1879634+JediLuke@users.noreply.github.com> Date: Fri, 4 Jul 2025 21:33:47 -0500 Subject: [PATCH 01/11] Auto docs --- guides/api_reference.md | 412 +++++++++++++++++++ guides/architecture_technical.md | 415 +++++++++++++++++++ guides/getting_started_architecture.md | 415 +++++++++++++++++++ guides/overview_scripts.md | 264 ++++++++++++ guides/overview_viewport.md | 141 ++++++- guides/troubleshooting.md | 537 +++++++++++++++++++++++++ lib/scenic/view_port.ex | 197 ++++++++- 7 files changed, 2368 insertions(+), 13 deletions(-) create mode 100644 guides/api_reference.md create mode 100644 guides/architecture_technical.md create mode 100644 guides/getting_started_architecture.md create mode 100644 guides/overview_scripts.md create mode 100644 guides/troubleshooting.md diff --git a/guides/api_reference.md b/guides/api_reference.md new file mode 100644 index 00000000..94dad2b4 --- /dev/null +++ b/guides/api_reference.md @@ -0,0 +1,412 @@ +# API Reference + +This reference provides detailed information about Scenic's core APIs, organized by module and use case. + +## Scenic.ViewPort + +The central coordinator managing scripts, input routing, and driver communication. + +### Core Functions + +#### `start/1` +```elixir +@spec start(opts :: Keyword.t()) :: {:ok, ViewPort.t()} +``` + +Start a new ViewPort process. + +**Options:** +- `:name` - Process name (optional) +- `:size` - `{width, height}` viewport dimensions (required) +- `:default_scene` - Root scene module or `{module, args}` (required) +- `:theme` - Theme configuration (default: `:dark`) +- `:drivers` - List of driver configurations (default: `[]`) +- `:input_filter` - Input type filter (default: `:all`) + +**Example:** +```elixir +{:ok, vp} = ViewPort.start([ + size: {800, 600}, + default_scene: MyApp.MainScene, + drivers: [[module: Scenic.Driver.Local]] +]) +``` + +#### `put_script/4` +```elixir +@spec put_script(ViewPort.t(), any, Script.t(), Keyword.t()) :: + {:ok, any} | :no_change | {:error, atom} +``` + +Store a compiled script and notify drivers. + +**Parameters:** +- `viewport` - ViewPort struct +- `name` - Script identifier (any term) +- `script` - Compiled script (list of commands) +- `opts` - Options (`:owner` pid) + +**Returns:** +- `{:ok, name}` - Script stored successfully +- `:no_change` - Identical script, no notification sent +- `{:error, reason}` - Storage error + +#### `put_graph/4` +```elixir +@spec put_graph(ViewPort.t(), any, Graph.t(), Keyword.t()) :: + {:ok, any} | {:error, atom} +``` + +Compile graph to script and store in ViewPort. + +**Parameters:** +- `viewport` - ViewPort struct +- `name` - Graph/script identifier +- `graph` - Graph to compile +- `opts` - Options (`:owner` pid) + +#### `get_script/2` +```elixir +@spec get_script(ViewPort.t(), any) :: {:ok, Script.t()} | {:error, :not_found} +``` + +Retrieve script by name from ETS table. + +#### `input/2` +```elixir +@spec input(ViewPort.t(), ViewPort.Input.t()) :: :ok | {:error, atom} +``` + +Send input event to ViewPort for processing. + +**Input Types:** +- `{:cursor_button, {button, action, mods, {x, y}}}` +- `{:cursor_pos, {x, y}}` +- `{:cursor_scroll, {offset, {x, y}}}` +- `{:key, {key, action, mods}}` +- `{:codepoint, {codepoint, mods}}` + +#### `start_driver/2` / `stop_driver/2` +```elixir +@spec start_driver(ViewPort.t(), Keyword.t()) :: {:ok, pid} | :error +@spec stop_driver(ViewPort.t(), pid) :: :ok +``` + +Dynamic driver management. + +### Scene Management + +#### `set_root/3` +```elixir +@spec set_root(ViewPort.t(), module, any) :: :ok +``` + +Change the root scene, stopping current scene hierarchy. + +#### `set_theme/2` +```elixir +@spec set_theme(ViewPort.t(), atom | map) :: :ok +``` + +Update the global theme and restart root scene. + +## Scenic.Script + +Low-level drawing command creation and manipulation. + +### Script Building + +#### `start/0` +```elixir +@spec start() :: Script.t() +``` + +Create empty script for building commands. + +#### `finish/1` +```elixir +@spec finish(Script.t()) :: Script.t() +``` + +Finalize script with optimizations. + +### Drawing Commands + +#### State Management +```elixir +push_state(script) # Save graphics state +pop_state(script) # Restore graphics state +reset_state(script) # Reset to initial state +``` + +#### Transforms +```elixir +translate(script, {x, y}) # Move coordinate system +scale(script, {sx, sy}) # Scale coordinate system +rotate(script, radians) # Rotate coordinate system +transform(script, matrix) # Apply matrix transform +``` + +#### Styling +```elixir +fill_color(script, color) # Set fill color +stroke_color(script, color) # Set stroke color +stroke_width(script, width) # Set line width +font(script, font_name) # Set font +font_size(script, size) # Set font size +``` + +#### Drawing Primitives +```elixir +draw_line(script, {from, to}) # Draw line +draw_rect(script, {width, height}) # Draw rectangle +draw_rrect(script, {w, h}, radius) # Draw rounded rectangle +draw_circle(script, radius) # Draw circle +draw_ellipse(script, {rx, ry}) # Draw ellipse +draw_text(script, text, {x, y}) # Draw text +draw_triangles(script, triangles) # Draw triangle mesh +draw_sprites(script, sprites) # Draw image sprites +``` + +#### Advanced +```elixir +draw_script(script, script_name) # Reference another script +draw_path(script, path_commands) # Draw vector path +scissor(script, {x, y, w, h}) # Set clipping rectangle +``` + +### Color Formats + +Colors can be specified as: +- Atoms: `:red`, `:blue`, `:transparent` +- RGB tuples: `{255, 128, 0}` +- RGBA tuples: `{255, 128, 0, 200}` +- Named colors: `{:color_rgb, {255, 128, 0}}` + +## Scenic.Driver + +Driver interface for rendering and input collection. + +### Driver Callbacks + +#### Required Callbacks + +```elixir +@callback validate_opts(Keyword.t()) :: {:ok, any} | {:error, String.t()} +@callback init(Driver.t(), Keyword.t()) :: {:ok, Driver.t()} +``` + +#### Optional Callbacks + +```elixir +@callback reset_scene(Driver.t()) :: {:ok, Driver.t()} +@callback request_input([Input.class()], Driver.t()) :: {:ok, Driver.t()} +@callback update_scene([Script.id()], Driver.t()) :: {:ok, Driver.t()} +@callback del_scripts([Script.id()], Driver.t()) :: {:ok, Driver.t()} +@callback clear_color(Color.t(), Driver.t()) :: {:ok, Driver.t()} +``` + +### Driver Helpers + +#### Input Sending +```elixir +send_input(driver, input_event) # Send input to ViewPort +``` + +#### State Management +```elixir +assign(driver, key, value) # Assign driver state +get(driver, key, default) # Get assigned value +set_busy(driver, boolean) # Set busy flag for batching +``` + +#### Update Control +```elixir +request_update(driver) # Request update_scene callback +``` + +### Input Event Format + +#### Mouse/Touch Input +```elixir +{:cursor_button, {button, action, modifiers, {x, y}}} +# button: :btn_left, :btn_right, :btn_middle, :btn_x1, :btn_x2 +# action: 0 (release), 1 (press) +# modifiers: [:ctrl, :shift, :alt, :meta] + +{:cursor_pos, {x, y}} +{:cursor_scroll, {offset, {x, y}}} +``` + +#### Keyboard Input +```elixir +{:key, {key, action, modifiers}} +# key: :space, :enter, :escape, :f1, etc. +# action: 0 (release), 1 (press), 2 (repeat) + +{:codepoint, {unicode_codepoint, modifiers}} +``` + +#### System Events +```elixir +{:viewport, {:reshape, {width, height}}} +{:viewport, :close} +``` + +## Scenic.Graph + +Declarative UI description and manipulation. + +### Graph Building + +#### `build/1` +```elixir +@spec build(opts :: Keyword.t()) :: Graph.t() +``` + +Create new graph with optional root styles. + +**Example:** +```elixir +Graph.build(font: :roboto, font_size: 16, fill: :white) +``` + +#### Primitive Addition +```elixir +# Import helpers from Scenic.Primitives +import Scenic.Primitives + +graph = Graph.build() +|> rectangle({100, 50}, fill: :blue, translate: {10, 20}) +|> text("Hello", font_size: 18, translate: {15, 35}) +|> circle(25, stroke: {2, :red}, translate: {50, 50}) +``` + +### Graph Modification + +#### `modify/3` +```elixir +@spec modify(Graph.t(), id, (Graph.t() -> Graph.t())) :: Graph.t() +``` + +Modify existing primitive by ID. + +**Example:** +```elixir +graph = Graph.build() +|> text("Counter: 0", id: :counter, translate: {10, 20}) + +# Later update the text +graph = Graph.modify(graph, :counter, &text(&1, "Counter: #{count}")) +``` + +#### `delete/2` +```elixir +@spec delete(Graph.t(), id) :: Graph.t() +``` + +Remove primitive by ID. + +### Component Integration + +#### `add_to_graph/3` +```elixir +# Add component scenes to graphs +graph = Graph.build() +|> MyComponent.add_to_graph(init_data, translate: {100, 100}) + +# Or using helper functions +import Scenic.Components +graph = Graph.build() +|> button("Click me", id: :my_button, translate: {50, 50}) +``` + +## Input Types Reference + +### Positional Input +Requires coordinate transformation and hit testing: +- `:cursor_button` - Mouse clicks/touches +- `:cursor_pos` - Mouse/touch movement +- `:cursor_scroll` - Scroll wheel/gestures + +### Non-Positional Input +Sent directly to requesting scenes: +- `:key` - Keyboard key presses +- `:codepoint` - Unicode character input +- `:viewport` - Window/viewport events + +### Input Modifiers +Available modifier keys: +- `:ctrl` - Control key +- `:shift` - Shift key +- `:alt` - Alt/Option key +- `:meta` - Windows/Cmd key + +### Button Types +Available mouse buttons: +- `:btn_left` - Primary button +- `:btn_right` - Secondary button +- `:btn_middle` - Middle button/wheel +- `:btn_x1` - Extra button 1 +- `:btn_x2` - Extra button 2 + +## Error Handling + +### Common Error Types + +#### ViewPort Errors +- `{:error, :invalid_size}` - Invalid viewport dimensions +- `{:error, :invalid_scene}` - Scene module doesn't exist or invalid +- `{:error, :driver_start_failed}` - Driver initialization failed + +#### Script Errors +- `{:error, :compilation_failed}` - Graph compilation error +- `{:error, :invalid_script}` - Malformed script commands +- `{:error, :not_found}` - Script doesn't exist + +#### Input Errors +- `{:error, :invalid_input}` - Malformed input event +- `{:error, :no_target}` - No scene to receive input + +### Error Recovery Patterns + +```elixir +# Graceful script compilation +case ViewPort.put_graph(viewport, name, graph) do + {:ok, _} -> + :ok + {:error, reason} -> + Logger.error("Graph compilation failed: #{inspect(reason)}") + # Use fallback graph or previous version + ViewPort.put_graph(viewport, name, fallback_graph) +end + +# Safe input handling +case ViewPort.input(viewport, input_event) do + :ok -> + :ok + {:error, :invalid_input} -> + Logger.warn("Invalid input ignored: #{inspect(input_event)}") +end +``` + +## Performance Guidelines + +### Script Optimization +- Cache compiled scripts when possible +- Use static scripts for unchanging content +- Break large graphs into smaller, reusable pieces +- Leverage change detection by avoiding unnecessary updates + +### Input Efficiency +- Request only needed input types +- Use input capture sparingly +- Implement proper input rate limiting in drivers + +### Memory Management +- Clean up script ownership properly +- Avoid creating many small scripts +- Monitor ETS table growth +- Use `:observer` to profile memory usage + +This API reference provides the essential interfaces for building Scenic applications. For implementation details and examples, see the other guides in this documentation. \ No newline at end of file diff --git a/guides/architecture_technical.md b/guides/architecture_technical.md new file mode 100644 index 00000000..49756d95 --- /dev/null +++ b/guides/architecture_technical.md @@ -0,0 +1,415 @@ +# Technical Architecture Guide + +This guide provides an in-depth technical explanation of how Scenic's core components work together to create a high-performance, fault-tolerant GUI framework. + +## Architectural Overview + +Scenic uses a layered architecture that cleanly separates concerns: + +``` +┌─────────────────────────────────────────────────────────────┐ +│ Applications │ +│ (Business Logic) │ +├─────────────────────────────────────────────────────────────┤ +│ Scenes │ +│ (UI Logic & State) │ +├─────────────────────────────────────────────────────────────┤ +│ Graphs │ +│ (Declarative UI) │ +├─────────────────────────────────────────────────────────────┤ +│ ViewPort │ +│ (Coordination Layer) │ +├─────────────────────────────────────────────────────────────┤ +│ Scripts │ +│ (Compiled Commands) │ +├─────────────────────────────────────────────────────────────┤ +│ Drivers │ +│ (Rendering & Input) │ +├─────────────────────────────────────────────────────────────┤ +│ Hardware │ +│ (Display & Input) │ +└─────────────────────────────────────────────────────────────┘ +``` + +## The ViewPort: Central Coordination + +### Core Architecture + +The ViewPort acts as the central nervous system, coordinating all interactions between the application layer and the rendering layer. + +#### ETS Table Strategy + +```elixir +# Script table - public, read-optimized for concurrent driver access +script_table = :ets.new(:_vp_script_table_, [ + :public, + {:read_concurrency, true} +]) + +# Table structure: {name, script, owner_pid} +:ets.insert(script_table, {"button_1", compiled_script, scene_pid}) +``` + +**Why ETS?** +- **Concurrent Access**: Multiple drivers can read simultaneously without blocking +- **Performance**: Direct memory access, no process serialization +- **Fault Tolerance**: Automatic cleanup when processes crash +- **Change Detection**: Efficient comparison of script content + +#### Process Monitoring + +```elixir +# ViewPort monitors all scenes and drivers +monitors = %{ + scene_pid => monitor_ref, + driver_pid => monitor_ref +} + +# Automatic cleanup on process crash +def handle_info({:DOWN, _ref, :process, pid, _reason}, state) do + # Clean up scripts owned by crashed process + :ets.match_delete(script_table, {:_, :_, pid}) + # Clean up input routing for crashed process + state = clean_input_state(state, pid) + {:noreply, state} +end +``` + +### Input Processing Architecture + +#### Hit Testing Algorithm + +```elixir +def input_find_hit(lists, input_type, name, {gx, gy} = global_point, parent_tx) do + case Map.fetch(lists, name) do + {:ok, {input_list, _, _}} -> + # Walk input list in reverse order (last drawn = first hit) + do_find_hit(input_list, input_type, global_point, lists, name, parent_tx) + _ -> + :not_found + end +end + +defp do_find_hit([{module, data, local_tx, pid, types, id} | tail], input_type, {gx, gy}, lists, name, parent_tx) do + # Calculate cumulative transform + combined_tx = Math.Matrix.mul(parent_tx, local_tx) + inverse_tx = Math.Matrix.invert(combined_tx) + + # Project global point to local coordinates + {x, y} = Math.Vector2.project({gx, gy}, inverse_tx) + + # Test if point is within primitive and input type matches + with true <- input_type == :any || Enum.member?(types, input_type), + true <- module.contains_point?(data, {x, y}) do + # Hit! Convert back to parent coordinate space + parent_xy = Math.Vector2.project({gx, gy}, Math.Matrix.invert(parent_tx)) + {:ok, pid, parent_xy, inverse_tx, id} + else + false -> do_find_hit(tail, input_type, {gx, gy}, lists, name, parent_tx) + end +end +``` + +#### Input Routing States + +```elixir +# Normal operation - scenes request input types +input_requests = %{ + :cursor_button => [scene_pid_1, scene_pid_2], + :key => [text_input_pid] +} + +# Temporary capture - one scene captures all input of a type +input_captures = %{ + :cursor_pos => [dragging_scene_pid], # Captures during drag + :key => [modal_scene_pid] # Captures during modal +} + +# Positional input types from all scenes +input_positional = [:cursor_button, :cursor_scroll, :cursor_pos] +``` + +## Script Compilation Pipeline + +### Graph → Script Transformation + +```elixir +# 1. Graph describes WHAT to draw +graph = Graph.build() +|> rectangle({100, 50}, fill: :blue, translate: {10, 20}) +|> text("Hello", font_size: 16, translate: {15, 35}) + +# 2. Compiler generates HOW to draw +script = GraphCompiler.compile(graph) +# Result: [ +# {:push_state}, +# {:fill_color, {:color_rgba, {0, 0, 255, 255}}}, +# {:translate, {10, 20}}, +# {:draw_rect, {100, 50}}, +# {:translate, {5, 15}}, # Relative offset +# {:font_size, 16}, +# {:draw_text, "Hello"}, +# {:pop_state} +# ] +``` + +### Compilation Optimizations + +#### Transform Flattening +```elixir +# Graph hierarchy: +# Group(translate: {10, 20}, scale: 2.0) +# └─ Rectangle(translate: {5, 5}) + +# Compiled to single transform: +# translate({10, 20}) → scale(2.0) → translate({5, 5}) +# Result: combined matrix applied once +combined_tx = Math.Matrix.mul([ + Matrix.translate({10, 20}), + Matrix.scale({2.0, 2.0}), + Matrix.translate({5, 5}) +]) +``` + +#### State Management +```elixir +# Automatic state push/pop insertion +script = [ + {:push_state}, # Preserve parent state + {:stroke_width, 2}, + {:stroke_color, :red}, + {:draw_line, {{0, 0}, {10, 10}}}, + {:pop_state} # Restore parent state +] +``` + +## Driver Architecture + +### Driver Lifecycle + +```elixir +defmodule Scenic.Driver do + # 1. Driver starts and registers with ViewPort + def init(driver, opts) do + GenServer.cast(viewport.pid, {:register_driver, self()}) + {:ok, driver} + end + + # 2. ViewPort sends current state + def handle_info({:_put_scripts_, ids}, driver) do + # Read scripts from ETS and render + scripts = Enum.map(ids, &ViewPort.get_script(viewport, &1)) + render_scripts(scripts, driver) + {:noreply, driver} + end + + # 3. Driver sends input back to ViewPort + def handle_mouse_click(button, x, y, driver) do + input = {:cursor_button, {button, 1, [], {x, y}}} + ViewPort.input(viewport, input) + {:noreply, driver} + end +end +``` + +### Rendering Pipeline + +```elixir +# Driver processes script commands sequentially +def render_script([cmd | rest], graphics_context) do + case cmd do + {:fill_color, color} -> + set_fill_color(graphics_context, color) + {:translate, {x, y}} -> + translate_context(graphics_context, x, y) + {:draw_rect, {w, h}} -> + draw_rectangle(graphics_context, w, h) + {:push_state} -> + push_graphics_state(graphics_context) + {:pop_state} -> + pop_graphics_state(graphics_context) + end + render_script(rest, graphics_context) +end +``` + +## Performance Characteristics + +### Concurrent Script Access + +```elixir +# Multiple drivers can read the same script simultaneously +# No serialization through ViewPort process + +Driver A: ViewPort.get_script(vp, "scene_1") # Direct ETS read +Driver B: ViewPort.get_script(vp, "scene_1") # Concurrent ETS read +Driver C: ViewPort.get_script(vp, "scene_1") # No blocking +``` + +### Change Detection Optimization + +```elixir +def put_script(viewport, name, new_script, opts) do + case :ets.lookup(script_table, name) do + [{_, ^new_script, _}] -> + :no_change # Identical script, no driver notification + _ -> + :ets.insert(script_table, {name, new_script, owner}) + notify_drivers({:put_scripts, [name]}) # Only notify on change + {:ok, name} + end +end +``` + +### Input Rate Limiting + +```elixir +# Driver-level input buffering +def send_input(driver, {:cursor_pos, _} = input) when driver.limit_ms > 0 do + case driver.input_limited do + true -> + # Buffer the input, send later + %{driver | input_buffer: Map.put(driver.input_buffer, :cursor_pos, input)} + false -> + # Send immediately and start rate limit timer + Process.send_after(self(), :_input_limiter_, driver.limit_ms) + send_input_now(driver, input) + end +end +``` + +## Fault Tolerance Mechanisms + +### Process Isolation + +```elixir +# Each component runs in its own process +Supervision Tree: +├─ Scenic.ViewPort (coordinator) +├─ DynamicSupervisor (scenes) +│ ├─ Scene.MainMenu +│ ├─ Scene.Settings +│ └─ Component.Button +└─ DynamicSupervisor (drivers) + ├─ Driver.Local + └─ Driver.Network +``` + +### Graceful Degradation + +```elixir +# ViewPort continues operating with partial failures +def handle_info({:DOWN, _ref, :process, driver_pid, reason}, state) do + Logger.warn("Driver #{inspect(driver_pid)} crashed: #{inspect(reason)}") + + # Remove from driver list but continue serving other drivers + state = %{state | driver_pids: List.delete(state.driver_pids, driver_pid)} + + # Scenes and other drivers unaffected + {:noreply, state} +end +``` + +### Automatic Resource Cleanup + +```elixir +# Scripts owned by crashed processes are automatically cleaned up +def handle_info({:DOWN, _ref, :process, pid, _reason}, state) do + # Remove all scripts owned by crashed process + :ets.match_delete(script_table, {:_, :_, pid}) + + # Clean up input routing + state = remove_input_requests(state, pid) + state = remove_input_captures(state, pid) + + {:noreply, state} +end +``` + +## Memory Management + +### Script Lifecycle + +```elixir +# Scripts are reference counted through ownership +script_owners = %{ + "button_1" => scene_pid_1, + "background" => scene_pid_1, + "modal" => scene_pid_2 +} + +# When scene crashes, all its scripts are cleaned up +# When script is replaced, old version is garbage collected +# ETS tables are memory-mapped and efficient +``` + +### Graph Compilation Caching + +```elixir +# Scenes can cache compiled results +defmodule MyScene do + # Compile once at module load time + @static_graph Graph.build() + |> rectangle({800, 600}, fill: :background) + + @static_script GraphCompiler.compile(@static_graph) + + def init(scene, _param, _opts) do + # Use pre-compiled script - no runtime compilation cost + ViewPort.put_script(scene.viewport, :background, @static_script) + {:ok, scene} + end +end +``` + +## Debugging and Introspection + +### Script Analysis + +```elixir +# Inspect compiled scripts +{:ok, script} = ViewPort.get_script(viewport, "my_scene") +Script.analyze(script) +# %{ +# command_count: 15, +# draw_calls: 3, +# state_changes: 8, +# transforms: 2, +# estimated_render_time: "0.1ms" +# } +``` + +### ViewPort State Inspection + +```elixir +# View current ViewPort state +{:ok, info} = ViewPort.info(viewport) +%ViewPort{ + name: :main_viewport, + size: {800, 600}, + script_table: #Reference<0.1234.5678>, + pid: #PID<0.123.0> +} + +# List all script IDs +ViewPort.all_script_ids(viewport) +# ["scene_1", "button_1", "background", ...] +``` + +### Input Flow Debugging + +```elixir +# Trace input routing +def handle_input({:cursor_button, {button, action, mods, {x, y}}}, state) do + Logger.debug("Input: button=#{button}, pos={#{x}, #{y}}") + + case input_find_hit(state.input_lists, :cursor_button, :_root_, {x, y}) do + {:ok, pid, local_xy, _tx, id} -> + Logger.debug("Hit: scene=#{inspect(pid)}, id=#{id}, local_pos=#{inspect(local_xy)}") + :not_found -> + Logger.debug("No hit found") + end +end +``` + +This architecture enables Scenic to achieve its design goals of high performance, fault tolerance, and clean separation of concerns while handling the complex coordination required for real-time GUI applications. \ No newline at end of file diff --git a/guides/getting_started_architecture.md b/guides/getting_started_architecture.md new file mode 100644 index 00000000..e448613e --- /dev/null +++ b/guides/getting_started_architecture.md @@ -0,0 +1,415 @@ +# Getting Started with Scenic Architecture + +This guide provides a hands-on introduction to understanding and working with Scenic's core architecture. By the end of this guide, you'll understand how ViewPorts, Drivers, Scripts, and Scenes work together. + +## Prerequisites + +- Basic Elixir knowledge (processes, GenServers, supervision) +- Understanding of GUI concepts (rendering, input handling) +- Familiarity with ETS tables (helpful but not required) + +## Quick Architecture Overview + +Before diving in, let's understand the key players: + +- **Scene**: Your application logic - creates graphs describing what to draw +- **Graph**: Declarative description of UI (like HTML) +- **ViewPort**: Central coordinator - compiles graphs to scripts, routes input +- **Script**: Compiled drawing commands (like assembly for graphics) +- **Driver**: Renders scripts to screen, captures user input + +## Your First ViewPort + +Let's start by creating a ViewPort and examining what happens: + +```elixir +# Start a minimal ViewPort +{:ok, viewport} = Scenic.ViewPort.start([ + name: :learning_viewport, + size: {400, 300}, + default_scene: MyApp.SimpleScene +]) + +# Examine the ViewPort struct +IO.inspect(viewport, label: "ViewPort") +# %Scenic.ViewPort{ +# name: :learning_viewport, +# pid: #PID<0.123.0>, +# script_table: #Reference<0.1234.5678>, +# size: {400, 300} +# } +``` + +**What just happened?** + +1. ViewPort process started and registered itself +2. ETS table created for storing scripts +3. Default scene process started +4. Scene compiled its initial graph to a script +5. Script stored in ETS table +6. Any connected drivers would be notified + +## Understanding Scripts + +Let's create a simple script manually to understand what drivers actually receive: + +```elixir +alias Scenic.Script + +# Create a script manually +my_script = Script.start() +|> Script.push_state() # Save graphics state +|> Script.fill_color({255, 0, 0, 255}) # Set red fill +|> Script.translate({50, 50}) # Move origin +|> Script.draw_rect({100, 60}) # Draw rectangle +|> Script.pop_state() # Restore state +|> Script.finish() # Optimize and finalize + +IO.inspect(my_script, label: "Compiled Script") +# [ +# {:push_state}, +# {:fill_color, {:color_rgba, {255, 0, 0, 255}}}, +# {:translate, {50, 50}}, +# {:draw_rect, {100, 60}}, +# {:pop_state} +# ] + +# Store it in the ViewPort +{:ok, _} = Scenic.ViewPort.put_script(viewport, "red_box", my_script) +``` + +**Key Insights:** + +- Scripts are lists of simple drawing commands +- They're optimized for fast execution by drivers +- State management (push/pop) prevents style bleeding +- Commands are stateful - order matters + +## From Graph to Script + +Now let's see how the normal workflow works - graphs compiled to scripts: + +```elixir +alias Scenic.Graph +import Scenic.Primitives + +# Create a graph (declarative) +graph = Graph.build(font_size: 16) +|> rectangle({100, 60}, fill: :red, translate: {50, 50}) +|> text("Hello", translate: {60, 75}, fill: :white) + +# Compile it to see what drivers receive +{:ok, compiled_script} = Scenic.Graph.Compiler.compile(graph) +IO.inspect(compiled_script, label: "Graph → Script") + +# Store in ViewPort (this is what push_graph does internally) +{:ok, _} = Scenic.ViewPort.put_graph(viewport, "hello_scene", graph) +``` + +**Compare the Results:** + +The compiled graph script will be more complex than our manual script because: +- Font information is included +- Text rendering commands added +- Coordinate calculations optimized +- State management automatically inserted + +## Working with the ETS Table + +The ViewPort uses ETS tables for high-performance script storage. Let's explore: + +```elixir +# Get all script IDs +script_ids = Scenic.ViewPort.all_script_ids(viewport) +IO.inspect(script_ids, label: "All Scripts") + +# Retrieve a specific script +{:ok, script} = Scenic.ViewPort.get_script(viewport, "hello_scene") +IO.inspect(length(script), label: "Script command count") + +# Examine the ETS table directly (advanced) +table = viewport.script_table +:ets.tab2list(table) |> IO.inspect(label: "Raw ETS contents") +``` + +**Understanding the Output:** + +- Each entry: `{name, script, owner_pid}` +- Scripts owned by the process that created them +- Automatic cleanup when owner crashes + +## Input Handling Fundamentals + +Let's explore how input flows through the system: + +```elixir +# Create a graph with clickable elements +interactive_graph = Graph.build() +|> rectangle({100, 50}, + fill: :blue, + translate: {50, 50}, + input: [:cursor_button]) # Make it clickable +|> text("Click me", + translate: {55, 70}, + input: [:cursor_button]) # Also clickable + +# Push to ViewPort +{:ok, _} = Scenic.ViewPort.put_graph(viewport, "interactive", interactive_graph) + +# Simulate input (what a driver would send) +input_event = {:cursor_button, {:btn_left, 1, [], {75, 65}}} # Click at (75, 65) +Scenic.ViewPort.input(viewport, input_event) +``` + +**What happens during input processing:** + +1. Driver captures mouse click at global coordinates +2. ViewPort receives input event +3. Hit testing performed against scene hierarchy +4. Coordinates projected to local scene space +5. Event sent to target scene with local coordinates + +## Building a Simple Driver + +Let's create a minimal driver to see the other side of the equation: + +```elixir +defmodule DebugDriver do + use Scenic.Driver + + def validate_opts(_opts), do: {:ok, []} + + def init(driver, _opts) do + IO.puts("Driver started!") + {:ok, driver} + end + + def update_scene(script_ids, driver) do + IO.puts("Scripts to render: #{inspect(script_ids)}") + + # Get and examine each script + Enum.each(script_ids, fn id -> + case Scenic.ViewPort.get_script(driver.viewport, id) do + {:ok, script} -> + IO.puts("Script #{id}: #{length(script)} commands") + # In a real driver, you'd render these commands + _ -> + IO.puts("Script #{id}: not found") + end + end) + + {:ok, driver} + end + + def request_input(input_types, driver) do + IO.puts("Input types requested: #{inspect(input_types)}") + {:ok, driver} + end +end + +# Start the debug driver +{:ok, _driver_pid} = Scenic.ViewPort.start_driver(viewport, [ + module: DebugDriver +]) +``` + +**Watch the Output:** + +When you create or update graphs, you'll see the driver receive notifications about which scripts to render. + +## Exploring Scene Lifecycle + +Let's trace what happens when scenes start and manage their UI: + +```elixir +defmodule LearningScene do + use Scenic.Scene + alias Scenic.Graph + import Scenic.Primitives + + def init(scene, _param, _opts) do + IO.puts("Scene starting: #{inspect(self())}") + + # Build initial graph + graph = Graph.build(font_size: 18) + |> text("Learning Scene", translate: {10, 30}) + |> rectangle({200, 100}, stroke: {2, :green}, translate: {10, 50}) + + # Push to ViewPort (compiles to script) + scene = push_graph(scene, graph) + IO.puts("Graph pushed to ViewPort") + + # Schedule an update to see change detection + Process.send_after(self(), :update_text, 2000) + + {:ok, assign(scene, counter: 0)} + end + + def handle_info(:update_text, scene) do + # Update the graph + counter = scene.assigns.counter + 1 + + graph = scene.assigns.graph + |> Graph.modify(:dynamic_text, &text(&1, "Counter: #{counter}")) + + scene = scene + |> assign(counter: counter) + |> push_graph(graph) + + # Schedule next update + Process.send_after(self(), :update_text, 1000) + + {:noreply, scene} + end +end + +# Set this as the new default scene +Scenic.ViewPort.set_root(viewport, LearningScene) +``` + +**Observe the Behavior:** + +- Scene process starts +- Initial graph compiled and stored +- Updates only trigger script compilation if content changed +- ViewPort coordinates all the communication + +## Performance Insights + +Let's explore Scenic's performance characteristics: + +```elixir +# Measure script compilation time +large_graph = Graph.build() +large_graph = Enum.reduce(1..1000, large_graph, fn i, graph -> + graph |> rectangle({10, 10}, translate: {rem(i, 50) * 15, div(i, 50) * 15}) +end) + +{time_us, {:ok, script}} = :timer.tc(fn -> + Scenic.Graph.Compiler.compile(large_graph) +end) + +IO.puts("Compiled #{length(script)} commands in #{time_us}μs") + +# Test change detection +{time_us, result} = :timer.tc(fn -> + Scenic.ViewPort.put_script(viewport, "large_graph", script) +end) +IO.puts("First store took #{time_us}μs, result: #{inspect(result)}") + +{time_us, result} = :timer.tc(fn -> + Scenic.ViewPort.put_script(viewport, "large_graph", script) # Same script +end) +IO.puts("Second store took #{time_us}μs, result: #{inspect(result)}") +``` + +**Performance Lessons:** + +- Compilation has a cost - cache when possible +- Change detection is very fast +- ETS operations are microsecond-scale +- Large graphs should be broken into reusable pieces + +## Debugging Techniques + +Here are essential debugging approaches: + +```elixir +# 1. Inspect ViewPort state +{:ok, info} = Scenic.ViewPort.info(viewport) +IO.inspect(info) + +# 2. List all scripts +script_ids = Scenic.ViewPort.all_script_ids(viewport) +IO.inspect(script_ids) + +# 3. Examine a compiled script +{:ok, script} = Scenic.ViewPort.get_script(viewport, "_root_") +IO.inspect(script |> Enum.take(10), label: "First 10 commands") + +# 4. Find scenes in the process tree +scenes = Process.list() +|> Enum.filter(fn pid -> + case Process.info(pid, :dictionary) do + {:dictionary, dict} -> + Keyword.get(dict, :"$initial_call") |> to_string() |> String.contains?("Scene") + _ -> false + end +end) +IO.inspect(scenes, label: "Scene processes") + +# 5. Monitor script changes +:ets.match(viewport.script_table, {:"$1", :"$2", :"$3"}) +|> Enum.each(fn [name, script, owner] -> + IO.puts("Script: #{name}, Commands: #{length(script)}, Owner: #{inspect(owner)}") +end) +``` + +## Common Patterns + +### Static vs Dynamic Content + +```elixir +# Static content - compile once, reuse many times +@background_script Script.start() +|> Script.fill_color({240, 240, 240, 255}) +|> Script.draw_rect({800, 600}) +|> Script.finish() + +def init(scene, _param, _opts) do + # Push static background + ViewPort.put_script(scene.viewport, :background, @background_script) + + # Dynamic content graph references static script + graph = Graph.build() + |> script(:background) # Reference to static script + |> text("Dynamic content", id: :dynamic, translate: {10, 10}) + + scene = push_graph(scene, graph) + {:ok, scene} +end +``` + +### Efficient Updates + +```elixir +# Only update what changed +def handle_info({:update_status, new_status}, scene) do + # Modify only the specific element + graph = scene.assigns.graph + |> Graph.modify(:status_text, &text(&1, new_status)) + + scene = push_graph(scene, graph) + {:noreply, scene} +end +``` + +## Next Steps + +Now that you understand the core architecture: + +1. **Explore Driver Development**: Look at `scenic_driver_local` for a real driver implementation +2. **Study Script Commands**: Read the `Scenic.Script` module documentation +3. **Build Complex Scenes**: Create scenes with multiple components +4. **Profile Performance**: Use `:observer` to watch process behavior +5. **Implement Custom Primitives**: Extend Scenic with your own primitive types + +## Troubleshooting + +**Script not rendering?** +- Check if driver is connected: `ViewPort.all_script_ids(viewport)` +- Verify script compilation: `Graph.Compiler.compile(graph)` +- Ensure ViewPort received the script + +**Input not working?** +- Verify `:input` style is set on primitives +- Check coordinate spaces (global vs local) +- Confirm scene is requesting the input type + +**Performance issues?** +- Profile script compilation time +- Break large graphs into smaller pieces +- Use static scripts for unchanging content +- Monitor ETS table size + +The key to mastering Scenic is understanding this data flow: `Graph → Script → Driver → Screen` and `Input → ViewPort → Scene`. Everything else builds on these fundamentals. \ No newline at end of file diff --git a/guides/overview_scripts.md b/guides/overview_scripts.md new file mode 100644 index 00000000..67021a17 --- /dev/null +++ b/guides/overview_scripts.md @@ -0,0 +1,264 @@ +# Script Overview + +Scenic Scripts are the fundamental rendering data structure that drivers actually interpret and draw. They represent the compiled, optimized form of your scene graphs - essentially a list of low-level drawing commands that can be efficiently processed by rendering systems. + +## What is a Script? + +A Script is an immutable list of drawing operations that tells a driver exactly what to render. Think of it as "assembly language for graphics" - it's what your high-level scene graphs get compiled down to. + +```elixir +# High-level graph +graph = Graph.build() +|> rectangle({100, 50}, fill: :blue, translate: {10, 20}) +|> text("Hello", font_size: 16, translate: {15, 35}) + +# Gets compiled to script (simplified representation) +script = [ + {:push_state}, + {:set_fill, {:color_rgba, {0, 0, 255, 255}}}, + {:translate, {10, 20}}, + {:draw_rect, {100, 50}}, + {:translate, {5, 15}}, # relative to previous + {:set_font_size, 16}, + {:draw_text, "Hello"}, + {:pop_state} +] +``` + +## Why Scripts Exist + +### Performance Optimization +- **Pre-compilation**: Expensive graph traversal and transformation happens once, not every frame +- **Optimized Commands**: Scripts contain only the minimal operations needed to render +- **Memory Efficiency**: Scripts can be shared between multiple drivers without duplication + +### Update Isolation +Scripts enable efficient partial updates: +- **Static Content**: Large, unchanging parts of UI can be pre-compiled to scripts +- **Dynamic Content**: Only the changing parts need recompilation +- **Independent Updates**: Scripts can be updated without affecting graphs that reference them + +### Driver Simplification +- **Consistent Interface**: All drivers receive the same script format regardless of source +- **Reduced Complexity**: Drivers don't need to understand graph structures or scene hierarchies +- **Parallel Processing**: Multiple drivers can process the same script simultaneously + +## Script Creation Patterns + +### Automatic Compilation +Most scripts are created automatically when you push graphs: + +```elixir +# In a scene +def init(scene, _param, _opts) do + graph = Graph.build() + |> text("Hello World", translate: {100, 100}) + + scene = push_graph(scene, graph) # Automatically compiles to script + {:ok, scene} +end +``` + +### Manual Script Creation +For advanced use cases, you can create scripts directly: + +```elixir +alias Scenic.Script + +# Build a reusable script for a complex graphic +checkmark_script = + Script.start() + |> Script.push_state() + |> Script.stroke_width(3) + |> Script.stroke_color(:green) + |> Script.line({{5, 10}, {8, 13}}) + |> Script.line({{8, 13}, {15, 6}}) + |> Script.pop_state() + |> Script.finish() + +# Publish the script +scene = push_script(scene, checkmark_script, "checkmark") + +# Reference from a graph +graph = Graph.build() +|> script("checkmark", translate: {50, 50}) +``` + +## Script Commands + +Scripts contain a variety of low-level drawing commands: + +### State Management +- `push_state` / `pop_state` - Save/restore graphics state +- `reset_state` - Reset to initial state + +### Transforms +- `translate` - Move coordinate system +- `scale` - Scale coordinate system +- `rotate` - Rotate coordinate system +- `transform` - Apply arbitrary matrix transformation + +### Styling +- `fill_color` / `stroke_color` - Set colors +- `stroke_width` - Set line width +- `font` / `font_size` - Set text properties +- `scissor` - Set clipping rectangle + +### Drawing Operations +- `draw_line` - Draw line segments +- `draw_rect` / `draw_rrect` - Draw rectangles +- `draw_circle` / `draw_ellipse` - Draw circular shapes +- `draw_text` - Render text +- `draw_triangles` - Draw triangle meshes +- `draw_sprites` - Draw image sprites + +### Advanced Operations +- `draw_script` - Reference another script (composition) +- `draw_path` - Draw complex vector paths +- `gradient_*` - Set up gradient fills + +## Script Lifecycle + +### Compilation +1. Scene calls `push_graph(scene, graph)` +2. ViewPort calls `Scenic.Graph.Compiler.compile(graph)` +3. Compiler traverses graph hierarchy depth-first +4. Each primitive contributes its drawing commands +5. Transforms and styles are applied and inherited +6. Result is an optimized command list + +### Storage and Distribution +1. Compiled script stored in ViewPort's ETS table +2. Change detection prevents unnecessary updates +3. ViewPort notifies all connected drivers +4. Drivers read script from ETS table concurrently + +### Execution +1. Driver reads script from ETS table +2. Driver processes commands sequentially +3. Graphics state maintained during execution +4. Output rendered to screen/file/network + +## Performance Considerations + +### Script Reuse +Scripts are immutable and can be safely reused: +- Same script can be referenced by multiple graphs +- Scripts can be cached and reused across scene changes +- Complex graphics compiled once, drawn many times + +### Memory Management +- Scripts are garbage collected when no longer referenced +- ViewPort cleans up scripts when owning processes crash +- Large scripts can be broken into smaller, reusable pieces + +### Update Strategies +- **Full Recompile**: Simple but potentially expensive for large graphs +- **Partial Updates**: Modify scripts surgically for specific changes +- **Script Composition**: Combine static and dynamic scripts + +## Error Handling + +### Compilation Errors +- Invalid primitives or malformed graphs detected at compile time +- Detailed error messages indicate problematic graph elements +- Scene initialization fails gracefully with clear diagnostics + +### Runtime Errors +- Malformed scripts detected when drivers attempt to process them +- Drivers can skip invalid commands and continue processing +- ViewPort monitors driver health and can restart failed drivers + +## Advanced Patterns + +### Script Templating +Create parameterized scripts for reusable components: + +```elixir +def create_button_script(text, width, height) do + Script.start() + |> Script.push_state() + |> Script.fill_color({200, 200, 200, 255}) + |> Script.draw_rrect({width, height}, 5) + |> Script.fill_color({0, 0, 0, 255}) + |> Script.font_size(14) + |> Script.text_align(:center) + |> Script.draw_text(text, {width/2, height/2}) + |> Script.pop_state() + |> Script.finish() +end +``` + +### Multi-Layer Composition +Combine multiple scripts for complex graphics: + +```elixir +# Background layer - static +background_script = create_background_script() + +# Content layer - dynamic +content_script = create_content_script(data) + +# Overlay layer - interactive elements +overlay_script = create_overlay_script() + +# Combine in graph +graph = Graph.build() +|> script("background") +|> script("content") +|> script("overlay") +``` + +### Performance Profiling +Scripts can be analyzed for optimization opportunities: +- Command count and complexity +- State change frequency +- Redundant operations +- Memory usage patterns + +## Related Documentation + +- [ViewPort Overview](overview_viewport.html) - How scripts are managed and distributed +- [Driver Overview](overview_driver.html) - How drivers process scripts +- [Graph Overview](overview_graph.html) - The high-level representation that compiles to scripts +- [Primitives Overview](overview_primitives.html) - The building blocks that generate script commands + +## Example: Complete Script Workflow + +```elixir +defmodule MyApp.Scene.Dashboard do + use Scenic.Scene + alias Scenic.{Graph, Script} + import Scenic.Primitives + + # Static background compiled at module load time + @background_script Script.start() + |> Script.fill_color({240, 240, 240, 255}) + |> Script.draw_rect({800, 600}) + |> Script.finish() + + def init(scene, _param, _opts) do + # Push the static background script + scene = push_script(scene, @background_script, "background") + + # Create dynamic content graph + graph = Graph.build() + |> script("background") # Reference static script + |> text("Loading...", translate: {50, 50}, id: :status) + + scene = push_graph(scene, graph) + {:ok, scene} + end + + def handle_info({:data_update, data}, scene) do + # Update only the dynamic parts - background script unchanged + graph = scene.assigns.graph + |> Graph.modify(:status, &text(&1, "Data: #{data}")) + + scene = push_graph(scene, graph) + {:noreply, scene} + end +end +``` + +Scripts are the bridge between Scenic's high-level declarative API and the low-level rendering systems, providing both performance and flexibility for complex graphical applications. \ No newline at end of file diff --git a/guides/overview_viewport.md b/guides/overview_viewport.md index 6ab77537..3cad4275 100644 --- a/guides/overview_viewport.md +++ b/guides/overview_viewport.md @@ -1,5 +1,142 @@ # ViewPort Overview -Give an overview of a viewport here +The ViewPort is the central orchestrator in Scenic's architecture. It acts as a liaison between the application logic (Scenes) and the rendering/input systems (Drivers), coordinating the flow of information while keeping these layers completely decoupled. -Coming soon \ No newline at end of file +## Core Responsibilities + +### 1. Script Management +The ViewPort owns and manages ETS tables that store compiled scripts. When scenes push graphs, the ViewPort: +- Compiles graphs into optimized scripts using `Scenic.Graph.Compiler` +- Stores scripts in public ETS tables for concurrent access by drivers +- Notifies drivers when scripts are updated or deleted +- Manages script lifecycle and cleanup + +### 2. Input Processing and Routing +The ViewPort handles all user input from drivers and routes it to appropriate scenes: +- **Positional Input**: Clicks, scrolls, cursor movement - requires hit testing through the scene hierarchy +- **Non-Positional Input**: Key presses, window events - routed to root or capturing scenes +- **Input Capture**: Allows scenes to capture input types temporarily (e.g., drag operations) +- **Transform Projection**: Converts global coordinates to local scene coordinates + +### 3. Scene Lifecycle Management +- Monitors scene processes and cleans up on crashes +- Manages scene hierarchy and parent-child relationships +- Tracks scene transforms for coordinate space conversions +- Handles scene startup coordination with gate mechanisms + +### 4. Driver Coordination +- Manages multiple drivers running simultaneously +- Broadcasts script updates to all connected drivers +- Communicates input requirements and theme changes +- Provides driver lifecycle management + +## Architecture Patterns + +### Script Compilation Pipeline +``` +Scene Graph → Graph Compiler → Script → ETS Table → Driver +``` + +1. Scene creates/modifies a graph +2. Scene calls `push_graph` +3. ViewPort compiles graph to script via `Scenic.Graph.Compiler` +4. Script stored in ETS table with change detection +5. Drivers notified of script updates +6. Drivers read scripts and render + +### Input Processing Pipeline +``` +Driver Input → ViewPort → Hit Testing → Scene Routing → Event Delivery +``` + +1. Driver captures raw input (mouse, keyboard, etc.) +2. ViewPort receives input and determines type +3. For positional input: hit testing against scene hierarchy +4. Transform coordinates to target scene's local space +5. Route to appropriate scene(s) based on capture/request state + +### The ETS Table Strategy +The ViewPort uses public ETS tables to achieve high-performance concurrent access: +- **Script Table**: `[:public, {:read_concurrency, true}]` - Multiple drivers can read simultaneously +- Scenes write compiled scripts directly to avoid serializing through ViewPort +- Change detection prevents unnecessary driver notifications +- Ownership tracked for cleanup when processes crash + +## Key Concepts + +### Transform Hierarchy +The ViewPort maintains a transform hierarchy that mirrors the scene structure: +- Each scene has a cumulative transform from root to local space +- Coordinate projection uses matrix operations for precision +- Enables complex nested UI layouts with proper input handling + +### Input Capture vs Request +- **Input Request**: Normal operation - scenes register interest in input types +- **Input Capture**: Temporary override - one scene captures all input of a type +- Used for drag operations, modal dialogs, or focus management + +### Gating Mechanism +The ViewPort can "gate" drivers during scene transitions: +- Prevents flickering during complex scene startup +- Coordinates when multiple scenes are initializing +- Signals completion when scene hierarchy is stable + +## Performance Characteristics + +### Concurrent Script Access +- Multiple drivers read the same scripts simultaneously +- No serialization bottleneck at ViewPort +- Scripts are immutable once compiled + +### Change Detection +- Scripts only sent to drivers when actually changed +- Reduces network traffic for remote drivers +- Enables efficient update batching + +### Input Rate Limiting +- Optional input rate limiting prevents overwhelming scenes +- Configurable per-driver for different hardware capabilities +- Buffers and batches high-frequency input like mouse movement + +## Error Handling and Recovery + +### Process Monitoring +- ViewPort monitors all scene and driver processes +- Automatic cleanup of scripts and state on process crash +- Maintains system integrity even with buggy scenes + +### Graceful Degradation +- ViewPort continues operating with partial driver failures +- Scene crashes don't affect other scenes or drivers +- Input routing adapts to scene hierarchy changes + +## Related Documentation + +- [Driver Overview](overview_driver.html) - How drivers interact with ViewPort +- [Scene Structure](overview_scene.html) - How scenes use ViewPort services +- [Script Overview](overview_scripts.html) - The rendering data structure +- [Input Handling](overview_input.html) - Detailed input processing + +## Example: Simple ViewPort Setup + +```elixir +# Start a ViewPort +{:ok, viewport} = Scenic.ViewPort.start([ + name: :main_viewport, + size: {800, 600}, + default_scene: MyApp.MainScene, + drivers: [ + [module: Scenic.Driver.Local, window: [title: "My App"]] + ] +]) + +# Scene pushes a graph +scene = push_graph(scene, my_graph) # Compiles to script, stores in ETS + +# Driver automatically receives script update and renders + +# User clicks - driver sends input to ViewPort +# ViewPort performs hit testing and routes to correct scene +``` + +The ViewPort's design achieves Scenic's goals of high performance, fault tolerance, and clean architectural separation while handling the complex coordination required for real-time GUI applications. \ No newline at end of file diff --git a/guides/troubleshooting.md b/guides/troubleshooting.md new file mode 100644 index 00000000..76d0b280 --- /dev/null +++ b/guides/troubleshooting.md @@ -0,0 +1,537 @@ +# Troubleshooting Guide + +This guide helps you diagnose and fix common issues when working with Scenic's architecture. + +## General Debugging Workflow + +1. **Identify the Layer**: Is the issue in Scene logic, ViewPort coordination, or Driver rendering? +2. **Check Process Health**: Are all processes running? Any crashes in logs? +3. **Inspect Data Flow**: Are graphs compiling? Scripts reaching drivers? Input routing correctly? +4. **Examine State**: ViewPort state, ETS contents, scene assigns +5. **Add Logging**: Trace the data flow through the system + +## Nothing Renders / Blank Screen + +### Symptoms +- Window opens but shows nothing +- Expected graphics don't appear +- No visual output from scenes + +### Diagnostic Steps + +#### 1. Check if ViewPort Started +```elixir +# Verify ViewPort is running +{:ok, info} = Scenic.ViewPort.info(:my_viewport) +IO.inspect(info) + +# Check if process is alive +Process.alive?(info.pid) +``` + +#### 2. Check Scene Status +```elixir +# List running processes that look like scenes +scenes = Process.list() +|> Enum.filter(fn pid -> + case Process.info(pid, :dictionary) do + {:dictionary, dict} -> + Keyword.get(dict, :"$initial_call") + |> to_string() + |> String.contains?("Scene") + _ -> false + end +end) + +IO.inspect(scenes, label: "Scene processes") +``` + +#### 3. Check Script Generation +```elixir +# List all scripts in ViewPort +script_ids = Scenic.ViewPort.all_script_ids(viewport) +IO.inspect(script_ids, label: "Available scripts") + +# Check if root script exists +case Scenic.ViewPort.get_script(viewport, "_root_") do + {:ok, script} -> + IO.puts("Root script has #{length(script)} commands") + {:error, :not_found} -> + IO.puts("ERROR: No root script found!") +end +``` + +#### 4. Check Driver Status +```elixir +# Check ETS table for scripts +:ets.tab2list(viewport.script_table) +|> Enum.each(fn {name, script, owner} -> + IO.puts("Script: #{name}, Commands: #{length(script)}, Owner: #{inspect(owner)}") +end) +``` + +### Common Causes & Solutions + +#### Scene Init Failed +```elixir +# Scene crashed during init - check logs +def init(scene, param, opts) do + # Add logging to debug + IO.puts("Scene init called with param: #{inspect(param)}") + + graph = Graph.build() |> text("Hello") + scene = push_graph(scene, graph) + + IO.puts("Graph pushed successfully") + {:ok, scene} +catch + error -> + IO.puts("Scene init failed: #{inspect(error)}") + {:stop, error} +end +``` + +#### No Driver Connected +```elixir +# Check if any drivers are running +driver_pids = :sys.get_state(viewport.pid).driver_pids +case driver_pids do + [] -> IO.puts("ERROR: No drivers connected!") + pids -> IO.puts("Drivers: #{inspect(pids)}") +end +``` + +#### Graph Compilation Failed +```elixir +# Test graph compilation manually +graph = Graph.build() |> text("test") +case Scenic.Graph.Compiler.compile(graph) do + {:ok, script} -> + IO.puts("Compilation successful: #{length(script)} commands") + {:error, reason} -> + IO.puts("ERROR: Compilation failed: #{inspect(reason)}") +end +``` + +## Input Not Working + +### Symptoms +- Clicks don't register +- Keyboard input ignored +- Mouse movement not tracked + +### Diagnostic Steps + +#### 1. Check Input Configuration +```elixir +# Verify primitives have input enabled +graph = Graph.build() +|> rectangle({100, 50}, + fill: :blue, + translate: {50, 50}, + input: [:cursor_button]) # Must have this! +``` + +#### 2. Check Input Requests +```elixir +# See what input types are being requested +state = :sys.get_state(viewport.pid) +IO.inspect(state._input_requests, label: "Input requests") +IO.inspect(state.input_positional, label: "Positional input") +``` + +#### 3. Test Input Injection +```elixir +# Manually inject input to test routing +test_input = {:cursor_button, {:btn_left, 1, [], {75, 75}}} +result = Scenic.ViewPort.input(viewport, test_input) +IO.inspect(result, label: "Input result") +``` + +#### 4. Check Hit Testing +```elixir +# Test hit detection manually +{:ok, hit} = Scenic.ViewPort.find_point(viewport, {75, 75}) +IO.inspect(hit, label: "Hit test result") +``` + +### Common Causes & Solutions + +#### Missing Input Style +```elixir +# WRONG - no input specified +|> rectangle({100, 50}, fill: :blue) + +# CORRECT - input types specified +|> rectangle({100, 50}, fill: :blue, input: [:cursor_button]) +``` + +#### Wrong Coordinate Space +```elixir +# Check if coordinates are in correct space +def handle_input({:cursor_button, {btn, action, mods, {x, y}}}, id, scene) do + IO.puts("Click at local coords: {#{x}, #{y}}, element: #{id}") + # Coordinates are already in scene's local space + {:noreply, scene} +end +``` + +#### Scene Not Requesting Input +```elixir +# Scene must request input types it wants +def init(scene, _param, _opts) do + # Request input types + scene = Scene.request_input(scene, [:cursor_button, :key]) + # ... rest of init +end +``` + +## Performance Issues + +### Symptoms +- Slow rendering +- High CPU usage +- Memory growth +- Laggy input response + +### Diagnostic Steps + +#### 1. Profile Script Compilation +```elixir +# Measure compilation time +{time_us, {:ok, script}} = :timer.tc(fn -> + Scenic.Graph.Compiler.compile(large_graph) +end) +IO.puts("Compilation took #{time_us}μs for #{length(script)} commands") +``` + +#### 2. Check Script Sizes +```elixir +# Find large scripts +Scenic.ViewPort.all_script_ids(viewport) +|> Enum.map(fn id -> + {:ok, script} = Scenic.ViewPort.get_script(viewport, id) + {id, length(script)} +end) +|> Enum.sort_by(fn {_, size} -> size end, :desc) +|> Enum.take(10) +|> IO.inspect(label: "Largest scripts") +``` + +#### 3. Monitor Process Memory +```elixir +# Check ViewPort memory usage +info = Process.info(viewport.pid, [:memory, :message_queue_len]) +IO.inspect(info, label: "ViewPort process info") + +# Check ETS table size +table_info = :ets.info(viewport.script_table) +IO.inspect(table_info[:size], label: "Script table entries") +``` + +#### 4. Profile with Observer +```elixir +# Start observer to monitor system +:observer.start() +``` + +### Common Causes & Solutions + +#### Frequent Graph Rebuilds +```elixir +# INEFFICIENT - rebuilds entire graph +def update_counter(scene, count) do + graph = Graph.build() + |> text("Count: #{count}") + |> button("Reset") + push_graph(scene, graph) +end + +# EFFICIENT - modify only what changed +def update_counter(scene, count) do + graph = scene.assigns.graph + |> Graph.modify(:counter, &text(&1, "Count: #{count}")) + push_graph(scene, graph) +end +``` + +#### Large Monolithic Graphs +```elixir +# INEFFICIENT - one huge graph +graph = Graph.build() +|> add_background() +|> add_menu() +|> add_content() +|> add_footer() + +# EFFICIENT - separate static and dynamic +# Use scripts for static content +ViewPort.put_script(vp, :background, background_script) +ViewPort.put_script(vp, :menu, menu_script) + +graph = Graph.build() +|> script(:background) +|> script(:menu) +|> add_dynamic_content() # Only this part recompiles +``` + +#### Memory Leaks +```elixir +# Check for orphaned scripts +:ets.tab2list(viewport.script_table) +|> Enum.map(fn {name, _script, owner} -> + alive = Process.alive?(owner) + {name, owner, alive} +end) +|> Enum.filter(fn {_, _, alive} -> not alive end) +|> IO.inspect(label: "Scripts with dead owners") +``` + +## Driver Issues + +### Symptoms +- Graphics appear but don't update +- Driver crashes or restarts +- Input works but no visual feedback + +### Diagnostic Steps + +#### 1. Check Driver Process +```elixir +# Find driver processes +driver_pids = :sys.get_state(viewport.pid).driver_pids +Enum.each(driver_pids, fn pid -> + case Process.info(pid) do + nil -> IO.puts("Driver #{inspect(pid)} is dead") + info -> IO.puts("Driver #{inspect(pid)} alive, memory: #{info[:memory]}") + end +end) +``` + +#### 2. Test Script Delivery +```elixir +# Create a simple test driver +defmodule TestDriver do + use Scenic.Driver + + def validate_opts(_), do: {:ok, []} + def init(driver, _), do: {:ok, driver} + + def update_scene(script_ids, driver) do + IO.puts("TestDriver received scripts: #{inspect(script_ids)}") + {:ok, driver} + end +end + +{:ok, _} = Scenic.ViewPort.start_driver(viewport, [module: TestDriver]) +``` + +#### 3. Check Driver Logs +Look for driver-specific error messages in logs. + +### Common Causes & Solutions + +#### Driver Not Handling Script Updates +```elixir +def update_scene(script_ids, driver) do + # Process each script + Enum.each(script_ids, fn id -> + case Scenic.ViewPort.get_script(driver.viewport, id) do + {:ok, script} -> render_script(script, driver) + {:error, :not_found} -> + Logger.warn("Script #{id} not found") + end + end) + {:ok, driver} +end +``` + +#### Graphics Context Issues +Ensure driver maintains proper graphics state: +```elixir +def render_script(script, driver) do + # Save initial state + save_graphics_state(driver.context) + + try do + Enum.each(script, &execute_command(&1, driver)) + after + # Always restore state + restore_graphics_state(driver.context) + end +end +``` + +## Scene Lifecycle Issues + +### Symptoms +- Scenes don't start +- Components not appearing +- Scene crashes on updates + +### Diagnostic Steps + +#### 1. Check Scene Registration +```elixir +# Check scene hierarchy +state = :sys.get_state(viewport.pid) +IO.inspect(state.scenes_by_pid, label: "Scenes by PID") +IO.inspect(state.scenes_by_id, label: "Scenes by ID") +``` + +#### 2. Test Scene Init +```elixir +def init(scene, param, opts) do + IO.puts("Scene #{__MODULE__} init called") + IO.inspect(param, label: "Param") + IO.inspect(opts, label: "Opts") + + # Your scene init code here + {:ok, scene} +catch + kind, error -> + IO.puts("Scene init failed: #{kind} #{inspect(error)}") + {:stop, error} +end +``` + +#### 3. Check Component Communication +```elixir +def handle_event({:click, :my_button}, _from, scene) do + IO.puts("Button clicked in #{__MODULE__}") + {:noreply, scene} +end +``` + +### Common Causes & Solutions + +#### Scene Init Crashes +```elixir +# Add error handling to scene init +def init(scene, param, _opts) do + try do + graph = build_initial_graph(param) + scene = push_graph(scene, graph) + {:ok, scene} + rescue + error -> + Logger.error("Scene init failed: #{inspect(error)}") + # Return a minimal working scene + graph = Graph.build() |> text("Error loading scene") + scene = push_graph(scene, graph) + {:ok, scene} + end +end +``` + +#### Component Not Found +```elixir +# Verify component module exists and is compiled +Code.ensure_loaded?(MyApp.Component.Button) + +# Check component is properly added to graph +graph = Graph.build() +|> MyApp.Component.Button.add_to_graph("Click me", id: :my_btn) +``` + +## ETS Table Issues + +### Symptoms +- Scripts not persisting +- Memory errors +- Access violations + +### Diagnostic Steps + +#### 1. Check Table Health +```elixir +# Verify table exists and is accessible +table = viewport.script_table +:ets.info(table) |> IO.inspect(label: "Table info") + +# Check table permissions +:ets.info(table, :protection) |> IO.inspect(label: "Protection") +``` + +#### 2. Check Table Contents +```elixir +# List all entries +:ets.tab2list(table) +|> Enum.take(10) # First 10 entries +|> IO.inspect(label: "Table contents") +``` + +### Common Causes & Solutions + +#### Table Access Violations +ETS tables should be `:public` for concurrent driver access: +```elixir +# CORRECT - public table with read concurrency +:ets.new(:script_table, [:public, {:read_concurrency, true}]) +``` + +#### Memory Pressure +```elixir +# Monitor table size +:ets.info(table, :memory) |> IO.inspect(label: "Table memory (words)") + +# Clean up orphaned entries +:ets.match_delete(table, {:_, :_, :"$1"}) +# This would delete entries where owner is dead (advanced usage) +``` + +## Common Error Messages + +### "Scene process not found" +- Scene crashed during startup +- Check scene `init/3` function for errors +- Verify scene module exists and compiles + +### "Script compilation failed" +- Invalid graph structure +- Missing or malformed primitives +- Check graph building code + +### "Input routing failed" +- No scene has requested the input type +- Hit testing failed (no matching primitives) +- Check primitive input styles + +### "Driver not responding" +- Driver process crashed +- Driver not implementing required callbacks +- Check driver logs for errors + +## Debug Logging Setup + +Add comprehensive logging to trace issues: + +```elixir +# In config/dev.exs +config :logger, level: :debug + +# In your modules +require Logger + +def put_graph(scene, graph) do + Logger.debug("Pushing graph with #{map_size(graph.primitives)} primitives") + result = push_graph(scene, graph) + Logger.debug("Graph push result: #{inspect(result)}") + result +end +``` + +## Using Observer for System Monitoring + +```elixir +# Start observer +:observer.start() + +# Key things to monitor: +# 1. Process tree - are all processes running? +# 2. Memory usage - any processes using excessive memory? +# 3. ETS tables - size and access patterns +# 4. Message queues - any processes with message buildup? +``` + +The key to effective debugging is understanding the data flow: `Scene → Graph → ViewPort → Script → Driver → Screen` and `Input → ViewPort → Scene`. Most issues occur at the boundaries between these components. \ No newline at end of file diff --git a/lib/scenic/view_port.ex b/lib/scenic/view_port.ex index df961334..1d4fcdbd 100644 --- a/lib/scenic/view_port.ex +++ b/lib/scenic/view_port.ex @@ -222,11 +222,51 @@ defmodule Scenic.ViewPort do # -------------------------------------------------------- @doc """ - Start a new ViewPort + Start a new ViewPort process. + + Creates a new ViewPort that coordinates between scenes and drivers. The ViewPort + manages ETS tables for scripts, handles input routing, and provides the central + coordination point for Scenic applications. + + ## Options + + - `:name` - Optional atom name to register the ViewPort process + - `:size` - Required `{width, height}` tuple defining the viewport dimensions + - `:default_scene` - Required scene module or `{module, args}` tuple for the root scene + - `:theme` - Theme configuration (defaults to `:dark`) + - `:drivers` - List of driver configurations to start automatically + - `:input_filter` - Filter for input types (defaults to `:all`) + - `:opts` - Additional style and transform options for the root graph + + ## Returns + + - `{:ok, %ViewPort{}}` - ViewPort struct containing process info and ETS table references + - Raises exception on configuration errors + + ## Examples + + # Basic viewport + {:ok, vp} = ViewPort.start([ + name: :main_viewport, + size: {800, 600}, + default_scene: MyApp.MainScene + ]) + + # With driver and theme + {:ok, vp} = ViewPort.start([ + size: {1024, 768}, + default_scene: {MyApp.Scene.Dashboard, %{user_id: 123}}, + theme: :light, + drivers: [ + [module: Scenic.Driver.Local, window: [title: "My App"]] + ] + ]) + + ## Notes + + The ViewPort creates its own supervision tree and manages scene/driver lifecycles. + If the ViewPort crashes, all associated scenes and scripts will need to be rebuilt. """ - # the ViewPort has it's own supervision tree under the ViewPorts node - # first create it's dynamic supervisor. Then start the ViewPort - # process underneath, passing it's supervisor in as an parameter. @spec start(opts :: Keyword.t()) :: {:ok, ViewPort.t()} def start(opts) do opts = Enum.into(opts, []) @@ -261,7 +301,37 @@ defmodule Scenic.ViewPort do # -------------------------------------------------------- @doc """ - Retrieve a script + Retrieve a compiled script by name from the ViewPort's ETS table. + + Scripts are the compiled, optimized rendering instructions that drivers use to + draw graphics. This function allows drivers and debugging tools to access + stored scripts directly. + + ## Parameters + + - `viewport` - The ViewPort struct containing the script table reference + - `name` - The unique identifier for the script (can be any term) + + ## Returns + + - `{:ok, script}` - The compiled script as a list of drawing commands + - `{:error, :not_found}` - If no script exists with the given name + + ## Examples + + # Retrieve a script by name + case ViewPort.get_script(viewport, "my_button") do + {:ok, script} -> + # Process or inspect the script + IO.inspect(script, label: "Button script") + {:error, :not_found} -> + Logger.warn("Script 'my_button' not found") + end + + ## Notes + + This function reads directly from the ETS table without going through the + ViewPort process, making it very fast for concurrent access by multiple drivers. """ @spec get_script(viewport :: ViewPort.t(), name :: any) :: {:ok, Script.t()} | {:error, :not_found} @@ -278,9 +348,51 @@ defmodule Scenic.ViewPort do end @doc """ - Put a script by name. + Store a compiled script in the ViewPort's ETS table and notify drivers. + + This is the primary mechanism for publishing rendering instructions to drivers. + Scripts are stored with change detection - if the same script content is + submitted again, drivers won't be notified unnecessarily. + + ## Parameters + + - `viewport` - The ViewPort struct containing the script table reference + - `name` - Unique identifier for the script (can be any term) + - `script` - Compiled script as a list of drawing commands + - `opts` - Optional configuration (see options below) + + ## Options + + - `:owner` - Process pid that owns the script (defaults to `self()`) + + ## Returns + + - `{:ok, name}` - Script successfully stored and drivers notified + - `:no_change` - Script content unchanged, no driver notification sent + - `{:error, atom}` - Error during script validation or storage + + ## Examples + + # Store a manually created script + script = Script.start() + |> Script.fill_color({255, 0, 0, 255}) + |> Script.draw_rect({100, 50}) + |> Script.finish() + + case ViewPort.put_script(viewport, "red_button", script) do + {:ok, _} -> Logger.info("Script stored successfully") + :no_change -> Logger.debug("Script unchanged") + end - returns `{:ok, id}` + # Store with custom owner (useful for cleanup tracking) + ViewPort.put_script(viewport, "component_1", script, owner: component_pid) + + ## Notes + + - Scripts are automatically cleaned up when the owning process crashes + - Change detection prevents unnecessary driver updates for identical scripts + - Multiple drivers can read the same script concurrently from the ETS table + - The function will notify all connected drivers about the script update """ @spec put_script( viewport :: ViewPort.t(), @@ -288,7 +400,7 @@ defmodule Scenic.ViewPort do script :: Script.t(), opts :: Keyword.t() ) :: - {:ok, non_neg_integer} | {:error, atom} + {:ok, any} | :no_change | {:error, atom} def put_script( %ViewPort{pid: pid, script_table: script_table}, name, @@ -348,16 +460,79 @@ defmodule Scenic.ViewPort do end @doc """ - Put a graph by name. + Compile a graph into a script and store it in the ViewPort. + + This is the primary mechanism scenes use to publish their UI for rendering. + The graph is compiled into an optimized script using `Scenic.Graph.Compiler`, + stored in the ViewPort's ETS table, and drivers are notified to update their + rendering. Input handling data is also extracted and registered. + + ## Parameters + + - `viewport` - The ViewPort struct containing the script table reference + - `name` - Unique identifier for the graph/script (can be any term) + - `graph` - The Graph struct containing primitives, styles, and transforms + - `opts` - Optional configuration (see options below) + + ## Options + + - `:owner` - Process pid that owns the graph (defaults to `self()`) + + ## Returns + + - `{:ok, name}` - Graph successfully compiled, stored, and drivers notified + - `{:error, reason}` - Compilation or storage error + + ## Examples + + # Basic graph publishing from a scene + graph = Graph.build() + |> rectangle({100, 50}, fill: :blue, translate: {10, 20}) + |> text("Click me", translate: {15, 35}) + + {:ok, _} = ViewPort.put_graph(viewport, :my_scene, graph) + + # With input handling + graph = Graph.build() + |> button("Submit", id: :submit_btn, translate: {100, 100}) + |> text("Status: Ready", id: :status, translate: {100, 150}) + + ViewPort.put_graph(viewport, :form_scene, graph) + + # Scene-owned graph (typical pattern) + ViewPort.put_graph(viewport, scene_id, graph, owner: self()) + + ## Compilation Process + + 1. Graph traversed depth-first to collect primitives + 2. Transforms and styles calculated and inherited + 3. Drawing commands generated for each primitive + 4. Input handling data extracted for clickable elements + 5. Script optimized and stored with change detection + 6. Drivers notified if script content changed + 7. Input routing tables updated in ViewPort + + ## Performance Notes + + - Compilation is expensive - avoid frequent graph rebuilds when possible + - Change detection prevents unnecessary driver updates + - Input data compilation enables efficient hit testing + - Large graphs should consider breaking into smaller, reusable scripts + + ## Error Handling - This compiles the graph to a collection of scripts + Compilation can fail if the graph contains: + - Invalid primitive data + - Malformed transforms or styles + - Circular script references + - Resource references that can't be resolved """ @spec put_graph( viewport :: ViewPort.t(), name :: any, graph :: Graph.t(), opts :: Keyword.t() - ) :: {:ok, name :: any} + ) :: {:ok, name :: any} | {:error, atom} def put_graph(%ViewPort{pid: pid} = viewport, name, %Graph{} = graph, opts \\ []) do opts = opts From eb91e9aa5fc5dfdc270a6f6a4184bf09bf235114 Mon Sep 17 00:00:00 2001 From: JediLuke <1879634+JediLuke@users.noreply.github.com> Date: Sun, 13 Jul 2025 20:56:18 -0500 Subject: [PATCH 02/11] wip lol --- SEMANTIC_DEV_GUIDE.md | 255 +++++++++++++ lib/scenic.ex | 91 +++++ lib/scenic/component/semantic_overlay.ex | 195 ++++++++++ lib/scenic/dev_tools.ex | 218 +++++++++++ lib/scenic/dev_tools_enhanced.ex | 143 +++++++ lib/scenic/primitives.ex | 22 ++ lib/scenic/primitives/semantic_overlay.ex | 46 +++ lib/scenic/scene.ex | 15 + lib/scenic/semantic.ex | 76 ++++ lib/scenic/semantic/query.ex | 121 ++++++ lib/scenic/view_port.ex | 128 ++++++- scenic-semantic-handover.md | 445 ++++++++++++++++++++++ test/scenic/semantic_test.exs | 75 ++++ 13 files changed, 1828 insertions(+), 2 deletions(-) create mode 100644 SEMANTIC_DEV_GUIDE.md create mode 100644 lib/scenic/component/semantic_overlay.ex create mode 100644 lib/scenic/dev_tools.ex create mode 100644 lib/scenic/dev_tools_enhanced.ex create mode 100644 lib/scenic/primitives/semantic_overlay.ex create mode 100644 lib/scenic/semantic.ex create mode 100644 lib/scenic/semantic/query.ex create mode 100644 scenic-semantic-handover.md create mode 100644 test/scenic/semantic_test.exs diff --git a/SEMANTIC_DEV_GUIDE.md b/SEMANTIC_DEV_GUIDE.md new file mode 100644 index 00000000..ea1c18d4 --- /dev/null +++ b/SEMANTIC_DEV_GUIDE.md @@ -0,0 +1,255 @@ +# Scenic Semantic Layer Developer Guide + +The Scenic semantic layer provides tools to inspect and query GUI elements by their semantic meaning during development. This guide shows how to use these tools when developing with Flamelex/Quillex. + +## Quick Start + +### 1. In your IEx session (when running Flamelex) + +```elixir +# Import the dev tools +import Scenic.DevTools + +# See all semantic info +semantic() + +# See all text buffers +buffers() + +# See content of buffer 1 +buffer(1) + +# See all buttons +buttons() + +# Find elements by type +find(:menu) +find(:text_input) +``` + +### 2. Adding Semantic Annotations to Your Components + +When building components, add semantic annotations to make them queryable: + +```elixir +@graph Graph.build() + # Text buffer with semantic info + |> text(buffer_content, + id: :buffer_text, + semantic: Semantic.text_buffer(buffer_id: 1)) + + # Button with semantic info + |> rect({100, 40}, + id: :save_button, + semantic: Semantic.button("Save")) + + # Text input field + |> text("", + id: :search_field, + semantic: Semantic.text_input("search", placeholder: "Search...")) +``` + +### 3. Visual Semantic Overlay + +Add a visual overlay to see semantic information in real-time: + +```elixir +defmodule MyApp.Scene.Main do + use Scenic.Scene + + @graph Graph.build() + |> text("Hello World", semantic: Semantic.text_buffer(buffer_id: 1)) + |> semantic_overlay(viewport: viewport, enabled: false) + + # Toggle overlay with keyboard shortcut + def handle_input({:key, {"S", :meta, _}}, _context, state) do + # Cmd+S toggles semantic overlay + cast_children({:semantic_overlay, :toggle}) + {:noreply, state} + end +end +``` + +## REPL Commands Reference + +### `semantic(viewport_name, graph_key)` +Shows full semantic tree with all elements organized by type. + +```elixir +iex> semantic() +=== Semantic Tree for :main === +Total elements: 5 + +By type: + button: 2 elements + - :save_btn: %{type: :button, label: "Save"} + - :cancel_btn: %{type: :button, label: "Cancel"} + text_buffer: 1 element + - :buffer_1: %{type: :text_buffer, buffer_id: 1} +``` + +### `buffers()` +Lists all text buffers with content preview. + +```elixir +iex> buffers() +Text Buffers: +[1] "def hello do\\n :world\\nend" +[2] "# TODO: implement feature" +``` + +### `buffer(id)` +Shows full content of a specific buffer. + +```elixir +iex> buffer(1) +Buffer 1: +def hello do + :world +end +``` + +### `buttons()` +Lists all buttons with their labels. + +```elixir +iex> buttons() +Buttons: +- "Save" (id: :save_btn) +- "Cancel" (id: :cancel_btn) +- "Submit" (id: :submit_btn) +``` + +### `find(type)` +Find all elements of a specific semantic type. + +```elixir +iex> find(:menu) +Found 2 menu element(s): +- :file_menu: %{type: :menu, name: "File", orientation: :vertical} +- :edit_menu: %{type: :menu, name: "Edit", orientation: :vertical} +``` + +### `types()` +List all semantic types currently in use. + +```elixir +iex> types() +Semantic types in use: +- button (3 elements) +- text_buffer (2 elements) +- menu (2 elements) +- text_input (1 element) +``` + +### `raw_semantic()` +Get the raw semantic data structure for advanced queries. + +```elixir +iex> data = raw_semantic() +iex> data.by_type +%{ + button: [:save_btn, :cancel_btn], + text_buffer: [:buffer_1] +} +``` + +## Query API for Testing + +The semantic layer provides a query API for use in tests: + +```elixir +alias Scenic.Semantic.Query + +# Get buffer text +{:ok, content} = Query.get_buffer_text(viewport, 1) + +# Find button by label +{:ok, button} = Query.get_button_by_label(viewport, "Save") + +# Get all editable content +{:ok, editable} = Query.get_editable_content(viewport) + +# Find elements with custom filter +{:ok, elem} = Query.find_one(viewport, :text_buffer, fn elem -> + elem.semantic.buffer_id == 1 +end) +``` + +## Common Semantic Types + +The `Scenic.Semantic` module provides helpers for common patterns: + +- `Semantic.button(label)` - Clickable buttons +- `Semantic.text_buffer(buffer_id: id)` - Text editor buffers +- `Semantic.text_input(name, opts)` - Input fields +- `Semantic.menu(name, opts)` - Menu containers +- `Semantic.menu_item(label, opts)` - Menu items +- `Semantic.annotate(type, attrs)` - Generic annotations + +## Tips for Development + +1. **Always add semantic info to interactive elements** - Buttons, inputs, and editable text should have semantic annotations. + +2. **Use consistent buffer IDs** - If buffer 1 is your main editor, keep it consistent across sessions. + +3. **Toggle overlay with hotkey** - Add a keyboard shortcut to toggle the semantic overlay during development. + +4. **Query in tests** - Use `Query` module functions instead of parsing visual output. + +5. **Custom semantic types** - Create your own semantic types for domain-specific elements: + +```elixir +# Custom semantic annotation +|> rect({100, 100}, + semantic: %{ + type: :code_cell, + language: :elixir, + cell_id: 42 + }) +``` + +## Debugging Tips + +If semantic info isn't showing: + +1. Check the element has a `:semantic` option +2. Verify the viewport name (default is `:main_viewport`) +3. Check the graph key (default is `:main`) +4. Use `raw_semantic()` to see all data +5. Ensure the graph was pushed after adding semantic info + +## Example: Flamelex Development Session + +```elixir +# Start Flamelex +iex -S mix + +# Import dev tools +import Scenic.DevTools + +# Check what's in the editor +buffers() +# Text Buffers: +# [1] "defmodule MyModule do\\n def hello, do: :world\\nend" + +# Make a change programmatically +Flamelex.Buffer.insert_text(1, "\\n def goodbye, do: :bye") + +# Verify the change +buffer(1) +# Buffer 1: +# defmodule MyModule do +# def hello, do: :world +# def goodbye, do: :bye +# end + +# See all interactive elements +types() +# Semantic types in use: +# - text_buffer (1 element) +# - button (3 elements) +# - menu (2 elements) +``` + +This semantic layer makes Flamelex development much more interactive and testable! \ No newline at end of file diff --git a/lib/scenic.ex b/lib/scenic.ex index 2759c5fe..248ea303 100644 --- a/lib/scenic.ex +++ b/lib/scenic.ex @@ -84,4 +84,95 @@ defmodule Scenic do ] |> Supervisor.init(strategy: :one_for_one) end + + # ========================================================================= + # Developer Tools - Convenience delegates + + @doc """ + Display semantic information for the current viewport. + + Convenience function that delegates to `Scenic.DevTools.semantic/2`. + + ## Examples + + iex> Scenic.semantic() + === Semantic Tree for :main === + Total elements: 3 + ... + """ + defdelegate semantic(viewport_name \\ :main_viewport, graph_key \\ :main), + to: Scenic.DevTools + + @doc """ + Show all text buffers and their content. + + Convenience function that delegates to `Scenic.DevTools.buffers/2`. + + ## Examples + + iex> Scenic.buffers() + Text Buffers: + [1] "Hello, World!" + ... + """ + defdelegate buffers(viewport_name \\ :main_viewport, graph_key \\ :main), + to: Scenic.DevTools + + @doc """ + Show content of a specific buffer. + + Convenience function that delegates to `Scenic.DevTools.buffer/3`. + + ## Examples + + iex> Scenic.buffer(1) + Buffer 1: + Hello, World! + """ + defdelegate buffer(buffer_id, viewport_name \\ :main_viewport, graph_key \\ :main), + to: Scenic.DevTools + + @doc """ + Show all buttons in the viewport. + + Convenience function that delegates to `Scenic.DevTools.buttons/2`. + + ## Examples + + iex> Scenic.buttons() + Buttons: + - "Save" (id: :save_btn) + ... + """ + defdelegate buttons(viewport_name \\ :main_viewport, graph_key \\ :main), + to: Scenic.DevTools + + @doc """ + Find elements by semantic type. + + Convenience function that delegates to `Scenic.DevTools.find/3`. + + ## Examples + + iex> Scenic.find(:menu) + Found 1 menu element(s): + - :main_menu: %{type: :menu, name: "File"} + """ + defdelegate find(type, viewport_name \\ :main_viewport, graph_key \\ :main), + to: Scenic.DevTools + + @doc """ + List all semantic types in use. + + Convenience function that delegates to `Scenic.DevTools.types/2`. + + ## Examples + + iex> Scenic.types() + Semantic types in use: + - button (2 elements) + - text_buffer (1 element) + """ + defdelegate types(viewport_name \\ :main_viewport, graph_key \\ :main), + to: Scenic.DevTools end diff --git a/lib/scenic/component/semantic_overlay.ex b/lib/scenic/component/semantic_overlay.ex new file mode 100644 index 00000000..90ecd2f1 --- /dev/null +++ b/lib/scenic/component/semantic_overlay.ex @@ -0,0 +1,195 @@ +defmodule Scenic.Component.SemanticOverlay do + @moduledoc """ + A component that visualizes semantic information as an overlay on your scene. + + This is useful during development to see what semantic annotations are + available on different GUI elements. + + ## Usage + + Add to your scene's graph: + + @graph Graph.build() + |> semantic_overlay(viewport: viewport, enabled: true) + + Toggle visibility with: + + Scene.cast(scene_pid, {:semantic_overlay, :toggle}) + Scene.cast(scene_pid, {:semantic_overlay, :show}) + Scene.cast(scene_pid, {:semantic_overlay, :hide}) + """ + + use Scenic.Component + + alias Scenic.{Graph, ViewPort} + alias Scenic.Semantic.Query + + # Component callbacks + + @impl true + def init(opts, _scenic_opts) do + viewport = Keyword.fetch!(opts, :viewport) + enabled = Keyword.get(opts, :enabled, false) + graph_key = Keyword.get(opts, :graph_key, :main) + + state = %{ + viewport: viewport, + enabled: enabled, + graph_key: graph_key, + graph: Graph.build() + } + + if enabled do + Process.send_after(self(), :update_overlay, 100) + end + + {:ok, state, push: state.graph} + end + + @impl true + def handle_cast({:semantic_overlay, :toggle}, state) do + new_state = %{state | enabled: not state.enabled} + + if new_state.enabled and not state.enabled do + send(self(), :update_overlay) + end + + graph = if new_state.enabled do + build_overlay(new_state) + else + Graph.build() + end + + {:noreply, %{new_state | graph: graph}, push: graph} + end + + def handle_cast({:semantic_overlay, :show}, state) do + if not state.enabled do + send(self(), :update_overlay) + end + + new_state = %{state | enabled: true} + graph = build_overlay(new_state) + {:noreply, %{new_state | graph: graph}, push: graph} + end + + def handle_cast({:semantic_overlay, :hide}, state) do + new_state = %{state | enabled: false} + graph = Graph.build() + {:noreply, %{new_state | graph: graph}, push: graph} + end + + @impl true + def handle_info(:update_overlay, %{enabled: false} = state) do + {:noreply, state} + end + + def handle_info(:update_overlay, %{enabled: true} = state) do + graph = build_overlay(state) + + # Schedule next update + Process.send_after(self(), :update_overlay, 1000) + + {:noreply, %{state | graph: graph}, push: graph} + end + + # Private functions + + defp build_overlay(state) do + case ViewPort.get_semantic(state.viewport, state.graph_key) do + {:ok, info} -> + build_semantic_visualization(info) + {:error, _} -> + Graph.build() + |> Scenic.Primitives.text("No semantic info available", + font_size: 12, + translate: {10, 20}, + fill: :red + ) + end + end + + defp build_semantic_visualization(info) do + # Start with base graph + graph = Graph.build() + + # Add background for readability + graph = graph + |> Scenic.Primitives.rect({300, 400}, + fill: {:black, 200}, + translate: {10, 10} + ) + + # Add title + graph = graph + |> Scenic.Primitives.text("Semantic Overlay", + font_size: 16, + translate: {20, 30}, + fill: :white + ) + + # Add summary + elem_count = map_size(info.elements) + type_count = map_size(info.by_type) + + graph = graph + |> Scenic.Primitives.text("#{elem_count} elements, #{type_count} types", + font_size: 12, + translate: {20, 50}, + fill: :light_gray + ) + + # List elements by type + {graph, _y_offset} = + info.by_type + |> Enum.sort() + |> Enum.reduce({graph, 70}, fn {type, ids}, {g, y} -> + # Type header + g = g + |> Scenic.Primitives.text("#{type}:", + font_size: 14, + translate: {20, y}, + fill: :cyan + ) + + # List elements of this type + {g, y} = Enum.reduce(ids, {g, y + 20}, fn id, {g2, y2} -> + elem = Map.get(info.elements, id) + label = format_element(elem) + + g2 = g2 + |> Scenic.Primitives.text(label, + font_size: 11, + translate: {30, y2}, + fill: :white + ) + + {g2, y2 + 15} + end) + + {g, y + 10} + end) + + graph + end + + defp format_element(elem) do + case elem.semantic.type do + :button -> + "Button: \"#{elem.semantic.label}\"" + :text_buffer -> + content_preview = if elem.content do + String.slice(elem.content || "", 0, 20) + else + "(empty)" + end + "Buffer #{elem.semantic.buffer_id}: #{content_preview}" + :text_input -> + "Input: #{elem.semantic.name}" + :menu -> + "Menu: #{elem.semantic.name}" + _other -> + "#{inspect(elem.id)}: #{inspect(elem.semantic)}" + end + end +end \ No newline at end of file diff --git a/lib/scenic/dev_tools.ex b/lib/scenic/dev_tools.ex new file mode 100644 index 00000000..f980accf --- /dev/null +++ b/lib/scenic/dev_tools.ex @@ -0,0 +1,218 @@ +defmodule Scenic.DevTools do + @moduledoc """ + Developer tools for inspecting Scenic applications during development. + + Import this module in your IEx session for easy access to semantic + inspection and debugging tools. + + ## Usage in IEx + + iex> import Scenic.DevTools + iex> semantic() # Show semantic info for default viewport + iex> semantic(:my_viewport) # Show semantic info for named viewport + iex> buffers() # Show all text buffers + iex> buttons() # Show all buttons + """ + + alias Scenic.{ViewPort, Semantic} + alias Scenic.Semantic.Query + + @doc """ + Display semantic information for the current viewport. + + ## Examples + + iex> semantic() + === Semantic Tree for :main === + Total elements: 3 + + By type: + text_buffer: 1 element + - :buffer_1: %{type: :text_buffer, buffer_id: 1} + button: 2 elements + - :save_btn: %{type: :button, label: "Save"} + - :cancel_btn: %{type: :button, label: "Cancel"} + """ + def semantic(viewport_name \\ :main_viewport, graph_key \\ :main) do + with {:ok, viewport} <- get_viewport(viewport_name) do + ViewPort.inspect_semantic(viewport, graph_key) + else + error -> + IO.puts("Error: #{inspect(error)}") + :error + end + end + + @doc """ + Show all text buffers and their content. + + ## Examples + + iex> buffers() + Text Buffers: + [1] "Hello, World!" + [2] "def my_function do\\n :ok\\nend" + """ + def buffers(viewport_name \\ :main_viewport, graph_key \\ :main) do + with {:ok, viewport} <- get_viewport(viewport_name), + {:ok, buffers} <- Query.find_by_type(viewport, :text_buffer, graph_key) do + IO.puts("Text Buffers:") + Enum.each(buffers, fn buffer -> + content = buffer.content || "" + buffer_id = buffer.semantic.buffer_id + preview = String.slice(content, 0, 60) + preview = if String.length(content) > 60, do: preview <> "...", else: preview + IO.puts("[#{buffer_id}] #{inspect(preview)}") + end) + :ok + else + {:error, :not_found} -> + IO.puts("No text buffers found") + :ok + error -> + IO.puts("Error: #{inspect(error)}") + :error + end + end + + @doc """ + Show content of a specific buffer. + + ## Examples + + iex> buffer(1) + Buffer 1: + Hello, World! + """ + def buffer(buffer_id, viewport_name \\ :main_viewport, graph_key \\ :main) do + with {:ok, viewport} <- get_viewport(viewport_name), + {:ok, content} <- Query.get_buffer_text(viewport, buffer_id, graph_key) do + IO.puts("Buffer #{buffer_id}:") + IO.puts(content) + :ok + else + error -> + IO.puts("Error: #{inspect(error)}") + :error + end + end + + @doc """ + Show all buttons in the viewport. + + ## Examples + + iex> buttons() + Buttons: + - "Save" (id: :save_btn) + - "Cancel" (id: :cancel_btn) + """ + def buttons(viewport_name \\ :main_viewport, graph_key \\ :main) do + with {:ok, viewport} <- get_viewport(viewport_name), + {:ok, buttons} <- Query.get_buttons(viewport, graph_key) do + IO.puts("Buttons:") + Enum.each(buttons, fn button -> + label = button.semantic.label + id = button.id + IO.puts("- #{inspect(label)} (id: #{inspect(id)})") + end) + :ok + else + {:error, :not_found} -> + IO.puts("No buttons found") + :ok + error -> + IO.puts("Error: #{inspect(error)}") + :error + end + end + + @doc """ + Find elements by semantic type. + + ## Examples + + iex> find(:menu) + Found 1 menu element(s): + - :main_menu: %{type: :menu, name: "File"} + """ + def find(type, viewport_name \\ :main_viewport, graph_key \\ :main) do + with {:ok, viewport} <- get_viewport(viewport_name), + {:ok, elements} <- Query.find_by_type(viewport, type, graph_key) do + IO.puts("Found #{length(elements)} #{type} element(s):") + Enum.each(elements, fn elem -> + IO.puts("- #{inspect(elem.id)}: #{inspect(elem.semantic)}") + end) + :ok + else + {:error, :not_found} -> + IO.puts("No #{type} elements found") + :ok + error -> + IO.puts("Error: #{inspect(error)}") + :error + end + end + + @doc """ + Get raw semantic data for a viewport. + + ## Examples + + iex> raw_semantic() + %{ + elements: %{...}, + by_type: %{...}, + timestamp: 1234567890 + } + """ + def raw_semantic(viewport_name \\ :main_viewport, graph_key \\ :main) do + with {:ok, viewport} <- get_viewport(viewport_name), + {:ok, info} <- ViewPort.get_semantic(viewport, graph_key) do + info + else + error -> error + end + end + + @doc """ + List all semantic types in use. + + ## Examples + + iex> types() + Semantic types in use: + - button (2 elements) + - text_buffer (1 element) + - menu (1 element) + """ + def types(viewport_name \\ :main_viewport, graph_key \\ :main) do + with {:ok, viewport} <- get_viewport(viewport_name), + {:ok, info} <- ViewPort.get_semantic(viewport, graph_key) do + IO.puts("Semantic types in use:") + info.by_type + |> Enum.sort_by(fn {_type, ids} -> -length(ids) end) + |> Enum.each(fn {type, ids} -> + count = length(ids) + element_word = if count == 1, do: "element", else: "elements" + IO.puts("- #{type} (#{count} #{element_word})") + end) + :ok + else + error -> + IO.puts("Error: #{inspect(error)}") + :error + end + end + + # Private helpers + + defp get_viewport(viewport) when is_struct(viewport, ViewPort), do: {:ok, viewport} + defp get_viewport(name) when is_atom(name) do + case Process.whereis(name) do + nil -> {:error, "ViewPort #{inspect(name)} not found"} + pid -> ViewPort.info(pid) + end + end + defp get_viewport(pid) when is_pid(pid), do: ViewPort.info(pid) +end \ No newline at end of file diff --git a/lib/scenic/dev_tools_enhanced.ex b/lib/scenic/dev_tools_enhanced.ex new file mode 100644 index 00000000..3b6e029c --- /dev/null +++ b/lib/scenic/dev_tools_enhanced.ex @@ -0,0 +1,143 @@ +defmodule Scenic.DevToolsEnhanced do + @moduledoc """ + Enhanced developer tools that work with UUID graph keys. + + This module extends the basic DevTools to handle cases where + graph keys are UUIDs rather than simple atoms. + """ + + alias Scenic.ViewPort + + @doc """ + Display all semantic information across all graphs. + """ + def semantic_all(viewport_name \\ :main_viewport) do + with {:ok, viewport} <- get_viewport(viewport_name) do + entries = :ets.tab2list(viewport.semantic_table) + + # Filter to only entries with semantic data + semantic_entries = Enum.filter(entries, fn {_key, data} -> + map_size(data.elements) > 0 + end) + + if semantic_entries == [] do + IO.puts("No semantic information found in any graph") + else + IO.puts("=== Semantic Information Across All Graphs ===") + IO.puts("Found #{length(semantic_entries)} graphs with semantic data\n") + + Enum.each(semantic_entries, fn {graph_key, data} -> + IO.puts("Graph: #{graph_key}") + IO.puts("Elements: #{map_size(data.elements)}") + + if map_size(data.by_type) > 0 do + IO.puts("By type:") + Enum.each(data.by_type, fn {type, ids} -> + IO.puts(" #{type}: #{length(ids)} element(s)") + + # Show details for each element + Enum.each(ids, fn id -> + elem = Map.get(data.elements, id) + case type do + :text_buffer -> + IO.puts(" - Buffer #{elem.semantic.buffer_id}") + content_preview = String.slice(elem.content || "", 0, 50) + if content_preview != "", do: IO.puts(" Content: #{inspect(content_preview)}") + + :button -> + IO.puts(" - Button: #{elem.semantic.label}") + + _ -> + IO.puts(" - #{inspect(elem.semantic)}") + end + end) + end) + end + IO.puts("") + end) + end + :ok + end + end + + @doc """ + Show all text buffers across all graphs. + """ + def buffers_all(viewport_name \\ :main_viewport) do + with {:ok, viewport} <- get_viewport(viewport_name) do + entries = :ets.tab2list(viewport.semantic_table) + + # Find all text buffers + all_buffers = Enum.flat_map(entries, fn {graph_key, data} -> + buffer_ids = Map.get(data.by_type, :text_buffer, []) + + Enum.map(buffer_ids, fn id -> + elem = Map.get(data.elements, id) + %{ + graph_key: graph_key, + buffer_id: elem.semantic.buffer_id, + content: elem.content || "", + semantic: elem.semantic + } + end) + end) + + if all_buffers == [] do + IO.puts("No text buffers found") + else + IO.puts("Text Buffers:") + Enum.each(all_buffers, fn buffer -> + content_preview = String.slice(buffer.content, 0, 60) + content_preview = if String.length(buffer.content) > 60, do: content_preview <> "...", else: content_preview + + IO.puts("\n[Buffer: #{buffer.buffer_id}]") + IO.puts("Graph: #{buffer.graph_key}") + IO.puts("Content: #{inspect(content_preview)}") + end) + end + :ok + end + end + + @doc """ + Get content of a specific buffer by UUID. + """ + def buffer_by_uuid(buffer_uuid, viewport_name \\ :main_viewport) do + with {:ok, viewport} <- get_viewport(viewport_name) do + entries = :ets.tab2list(viewport.semantic_table) + + # Search for the buffer + result = Enum.find_value(entries, fn {_graph_key, data} -> + buffer_ids = Map.get(data.by_type, :text_buffer, []) + + Enum.find_value(buffer_ids, fn id -> + elem = Map.get(data.elements, id) + if elem.semantic.buffer_id == buffer_uuid do + elem.content || "" + end + end) + end) + + case result do + nil -> + IO.puts("Buffer #{buffer_uuid} not found") + :error + content -> + IO.puts("Buffer #{buffer_uuid}:") + IO.puts(content) + :ok + end + end + end + + # Private helpers + + defp get_viewport(viewport) when is_struct(viewport, ViewPort), do: {:ok, viewport} + defp get_viewport(name) when is_atom(name) do + case Process.whereis(name) do + nil -> {:error, "ViewPort #{inspect(name)} not found"} + pid -> ViewPort.info(pid) + end + end + defp get_viewport(pid) when is_pid(pid), do: ViewPort.info(pid) +end \ No newline at end of file diff --git a/lib/scenic/primitives.ex b/lib/scenic/primitives.ex index 29efde50..16cd9692 100644 --- a/lib/scenic/primitives.ex +++ b/lib/scenic/primitives.ex @@ -1480,6 +1480,28 @@ defmodule Scenic.Primitives do def update_opts(p, opts), do: Primitive.merge_opts(p, opts) + # -------------------------------------------------------- + @doc """ + Add a semantic overlay component to a graph. + + The semantic overlay displays real-time information about semantic + annotations in your GUI, useful for development and debugging. + + ## Examples + + @graph Graph.build() + |> semantic_overlay(viewport: viewport) + + # With options + @graph Graph.build() + |> semantic_overlay( + viewport: viewport, + enabled: true, + translate: {500, 10} + ) + """ + defdelegate semantic_overlay(graph, opts \\ []), to: Scenic.Primitives.SemanticOverlay + # ============================================================================ # generic workhorse versions diff --git a/lib/scenic/primitives/semantic_overlay.ex b/lib/scenic/primitives/semantic_overlay.ex new file mode 100644 index 00000000..116f541f --- /dev/null +++ b/lib/scenic/primitives/semantic_overlay.ex @@ -0,0 +1,46 @@ +defmodule Scenic.Primitives.SemanticOverlay do + @moduledoc """ + Convenience functions for adding a semantic overlay to your scene. + + The semantic overlay displays real-time information about semantic + annotations in your GUI, useful for development and debugging. + """ + + alias Scenic.Graph + alias Scenic.Component.SemanticOverlay + + @doc """ + Add a semantic overlay component to a graph. + + ## Examples + + @graph Graph.build() + |> semantic_overlay(viewport: viewport) + + # With options + @graph Graph.build() + |> semantic_overlay( + viewport: viewport, + enabled: true, + translate: {500, 10} + ) + """ + def semantic_overlay(graph, opts \\ []) + + def semantic_overlay(%Graph{} = graph, opts) do + # Extract component options + viewport = Keyword.fetch!(opts, :viewport) + enabled = Keyword.get(opts, :enabled, false) + graph_key = Keyword.get(opts, :graph_key, :main) + + # Extract primitive options + styles = Keyword.take(opts, [:translate, :scale, :rotate, :pin, :hidden]) + + # Add the component + Graph.add_to_graph( + graph, + {SemanticOverlay, [viewport: viewport, enabled: enabled, graph_key: graph_key]}, + Keyword.merge([id: :semantic_overlay], styles) + ) + end +end \ No newline at end of file diff --git a/lib/scenic/scene.ex b/lib/scenic/scene.ex index c525441e..bef8135f 100644 --- a/lib/scenic/scene.ex +++ b/lib/scenic/scene.ex @@ -1512,6 +1512,21 @@ defmodule Scenic.Scene do end end + # def handle_call(:get_graph, _from, %Scene{assigns: assigns} = scene) do + # # Default implementation for graph introspection + # # Scenes can override this by implementing their own handle_call(:get_graph, ...) + # graph_info = %{ + # module: scene.module, + # id: scene.id, + # # Look for common graph storage patterns + # graph: Map.get(assigns, :graph), + # graphs: Map.get(assigns, :graphs), + # # Include any other potentially useful info + # assigns_keys: Map.keys(assigns) + # } + # {:reply, {:ok, graph_info}, scene} + # end + # -------------------------------------------------------- # generic handle_call. give the scene a chance to handle it def handle_call(msg, from, %Scene{module: module} = scene) do diff --git a/lib/scenic/semantic.ex b/lib/scenic/semantic.ex new file mode 100644 index 00000000..5a345b19 --- /dev/null +++ b/lib/scenic/semantic.ex @@ -0,0 +1,76 @@ +defmodule Scenic.Semantic do + @moduledoc """ + Semantic information helpers for Scenic components. + + Provides consistent semantic annotations for testing and accessibility. + """ + + @doc """ + Mark an element as a button. + + ## Examples + |> rect({100, 40}, semantic: Scenic.Semantic.button("Submit")) + """ + def button(label) do + %{type: :button, label: label, role: :button} + end + + @doc """ + Mark an element as an editable text buffer. + + ## Examples + |> text(content, semantic: Scenic.Semantic.text_buffer(buffer_id: 1)) + """ + def text_buffer(opts) do + %{ + type: :text_buffer, + buffer_id: Keyword.fetch!(opts, :buffer_id), + editable: Keyword.get(opts, :editable, true), + role: :textbox + } + end + + @doc """ + Mark an element as a text input field. + """ + def text_input(name, opts \\ []) do + %{ + type: :text_input, + name: name, + value: Keyword.get(opts, :value), + placeholder: Keyword.get(opts, :placeholder), + role: :textbox + } + end + + @doc """ + Mark an element as a menu. + """ + def menu(name, opts \\ []) do + %{ + type: :menu, + name: name, + orientation: Keyword.get(opts, :orientation, :vertical), + role: :menu + } + end + + @doc """ + Mark an element as a menu item. + """ + def menu_item(label, opts \\ []) do + %{ + type: :menu_item, + label: label, + parent_menu: Keyword.get(opts, :parent_menu), + role: :menuitem + } + end + + @doc """ + Generic semantic annotation. + """ + def annotate(type, attrs \\ %{}) do + Map.merge(%{type: type}, attrs) + end +end \ No newline at end of file diff --git a/lib/scenic/semantic/query.ex b/lib/scenic/semantic/query.ex new file mode 100644 index 00000000..c8318d29 --- /dev/null +++ b/lib/scenic/semantic/query.ex @@ -0,0 +1,121 @@ +defmodule Scenic.Semantic.Query do + @moduledoc """ + Query API for semantic information in Scenic ViewPorts. + + Provides testing-friendly functions to find and inspect GUI elements + based on their semantic meaning rather than visual properties. + """ + + @doc """ + Get semantic information for a graph. + """ + def get_semantic_info(viewport, graph_key \\ :_root_) do + case :ets.lookup(viewport.semantic_table, graph_key) do + [{^graph_key, info}] -> {:ok, info} + [] -> {:error, :no_semantic_info} + end + end + + @doc """ + Find all elements of a specific semantic type. + + ## Examples + Query.find_by_type(viewport, :button) + Query.find_by_type(viewport, :text_buffer) + """ + def find_by_type(viewport, type, graph_key \\ :_root_) do + with {:ok, info} <- get_semantic_info(viewport, graph_key) do + ids = get_in(info, [:by_type, type]) || [] + elements = Enum.map(ids, &Map.get(info.elements, &1)) + {:ok, elements} + end + end + + @doc """ + Find a single element by semantic type and additional filter. + + ## Examples + Query.find_one(viewport, :text_buffer, fn elem -> + elem.semantic.buffer_id == 1 + end) + """ + def find_one(viewport, type, filter_fn, graph_key \\ :_root_) do + with {:ok, elements} <- find_by_type(viewport, type, graph_key) do + case Enum.find(elements, filter_fn) do + nil -> {:error, :not_found} + element -> {:ok, element} + end + end + end + + @doc """ + Get text content from a text buffer by buffer_id. + """ + def get_buffer_text(viewport, buffer_id, graph_key \\ :_root_) do + with {:ok, buffer} <- find_one(viewport, :text_buffer, fn elem -> + elem.semantic.buffer_id == buffer_id + end, graph_key) do + {:ok, buffer.content || ""} + end + end + + @doc """ + Find all buttons in the viewport. + """ + def get_buttons(viewport, graph_key \\ :_root_) do + find_by_type(viewport, :button, graph_key) + end + + @doc """ + Find button by label. + """ + def get_button_by_label(viewport, label, graph_key \\ :_root_) do + find_one(viewport, :button, fn elem -> + elem.semantic.label == label + end, graph_key) + end + + @doc """ + Get all editable text content. + """ + def get_editable_content(viewport, graph_key \\ :_root_) do + with {:ok, info} <- get_semantic_info(viewport, graph_key) do + editable = + info.elements + |> Map.values() + |> Enum.filter(fn elem -> + get_in(elem, [:semantic, :editable]) == true + end) + |> Enum.map(fn elem -> + %{ + id: elem.id, + type: elem.semantic.type, + content: elem.content, + buffer_id: elem.semantic[:buffer_id] + } + end) + {:ok, editable} + end + end + + @doc """ + Debug helper - print all semantic elements. + """ + def inspect_semantic_tree(viewport, graph_key \\ :_root_) do + with {:ok, info} <- get_semantic_info(viewport, graph_key) do + IO.puts("=== Semantic Tree for #{graph_key} ===") + IO.puts("Total elements: #{map_size(info.elements)}") + IO.puts("\nBy type:") + + Enum.each(info.by_type, fn {type, ids} -> + IO.puts(" #{type}: #{length(ids)} elements") + Enum.each(ids, fn id -> + elem = Map.get(info.elements, id) + IO.puts(" - #{id}: #{inspect(elem.semantic)}") + end) + end) + + :ok + end + end +end \ No newline at end of file diff --git a/lib/scenic/view_port.ex b/lib/scenic/view_port.ex index 1d4fcdbd..3567e48a 100644 --- a/lib/scenic/view_port.ex +++ b/lib/scenic/view_port.ex @@ -114,12 +114,14 @@ defmodule Scenic.ViewPort do pid: pid, # name_table: reference, script_table: reference, + semantic_table: reference, size: {number, number} } defstruct name: nil, pid: nil, # name_table: nil, script_table: nil, + semantic_table: nil, size: nil @viewports :scenic_viewports @@ -533,7 +535,7 @@ defmodule Scenic.ViewPort do graph :: Graph.t(), opts :: Keyword.t() ) :: {:ok, name :: any} | {:error, atom} - def put_graph(%ViewPort{pid: pid} = viewport, name, %Graph{} = graph, opts \\ []) do + def put_graph(%ViewPort{pid: pid, semantic_table: semantic_table} = viewport, name, %Graph{} = graph, opts \\ []) do opts = opts |> Enum.into([]) @@ -557,6 +559,10 @@ defmodule Scenic.ViewPort do # write the script to the table # this notifies the drivers... put_script(viewport, name, script, owner: owner) + + # Build and store semantic information + semantic_info = build_semantic_info(graph, name) + true = :ets.insert(semantic_table, {name, semantic_info}) # send the input list to the viewport GenServer.cast(pid, {:input_list, input_list, name, owner}) @@ -659,6 +665,53 @@ defmodule Scenic.ViewPort do GenServer.call(pid, {:stop_driver, driver_pid}) end + # -------------------------------------------------------- + @doc """ + Get the semantic information for a graph in the viewport. + + This is useful during development to inspect what semantic annotations + are available for testing. + + ## Examples + + iex> ViewPort.get_semantic(viewport) + {:ok, %{elements: %{...}, by_type: %{...}}} + + iex> ViewPort.get_semantic(viewport, :specific_graph) + {:ok, %{...}} + """ + @spec get_semantic(viewport :: ViewPort.t(), graph_key :: any) :: + {:ok, map} | {:error, :no_semantic_info} + def get_semantic(%ViewPort{pid: pid}, graph_key \\ :main) do + GenServer.call(pid, {:get_semantic, graph_key}) + end + + # -------------------------------------------------------- + @doc """ + Inspect all semantic information in the viewport. + + This prints a formatted view of all semantic elements, making it easy + to see what's available during development. + + ## Examples + + iex> ViewPort.inspect_semantic(viewport) + === Semantic Tree for :main === + Total elements: 5 + + By type: + button: 2 elements + - :submit_btn: %{type: :button, label: "Submit"} + - :cancel_btn: %{type: :button, label: "Cancel"} + text_buffer: 1 element + - :buffer_1: %{type: :text_buffer, buffer_id: 1} + :ok + """ + @spec inspect_semantic(viewport :: ViewPort.t(), graph_key :: any) :: :ok + def inspect_semantic(%ViewPort{} = viewport, graph_key \\ :main) do + Scenic.Semantic.Query.inspect_semantic_tree(viewport, graph_key) + end + # -------------------------------------------------------- @doc false def start_link(opts) do @@ -677,6 +730,7 @@ defmodule Scenic.ViewPort do # script_table = :ets.new( make_ref(), [:public, {:read_concurrency, true}] ) # name_table = :ets.new(:_vp_name_table_, [:protected]) script_table = :ets.new(:_vp_script_table_, [:public, {:read_concurrency, true}]) + semantic_table = :ets.new(:_vp_semantic_table_, [:public, {:read_concurrency, true}]) state = %{ # simple metadata about the ViewPort @@ -714,6 +768,7 @@ defmodule Scenic.ViewPort do # becomes problematic, the next step is to have the scripts compile, then send # finished scripts to the VP for writing. script_table: script_table, + semantic_table: semantic_table, # state related to input from drivers to scenes # input lists are generated when a scene pushes a graph. Primitives @@ -1251,6 +1306,16 @@ defmodule Scenic.ViewPort do {:reply, :_pong_, scene} end + # -------------------------------------------------------- + # Semantic information access + def handle_call({:get_semantic, graph_key}, _from, %{semantic_table: semantic_table} = state) do + result = case :ets.lookup(semantic_table, graph_key) do + [{^graph_key, info}] -> {:ok, info} + [] -> {:error, :no_semantic_info} + end + {:reply, result, state} + end + def handle_call(invalid, from, %{name: name} = state) do Logger.error(""" ViewPort #{inspect(name || self())} ignored bad call @@ -1346,6 +1411,7 @@ defmodule Scenic.ViewPort do name: name, # name_table: name_table, script_table: script_table, + semantic_table: semantic_table, size: size }) do %ViewPort{ @@ -1353,6 +1419,7 @@ defmodule Scenic.ViewPort do name: name, # name_table: name_table, script_table: script_table, + semantic_table: semantic_table, size: size } end @@ -1399,7 +1466,7 @@ defmodule Scenic.ViewPort do defp internal_put_graph( %Graph{} = graph, name, - %{input_lists: ils, script_table: script_table} = state + %{input_lists: ils, script_table: script_table, semantic_table: semantic_table} = state ) do state = with {:ok, script} <- GraphCompiler.compile(graph), @@ -1413,6 +1480,11 @@ defmodule Scenic.ViewPort do # it isn't there or has changed _ -> true = :ets.insert(script_table, {name, script, :viewport}) + + # Build and store semantic information + semantic_info = build_semantic_info(graph, name) + true = :ets.insert(semantic_table, {name, semantic_info}) + :ok end @@ -1995,4 +2067,56 @@ defmodule Scenic.ViewPort do do_find_hit(tail, input_type, gp, lists, name, parent_tx) end end + + # Build semantic information from a graph + defp build_semantic_info(graph, graph_key) do + elements = + graph.primitives + |> Enum.reduce(%{}, fn {id, primitive}, acc -> + # Extract semantic data if present - use direct access for struct fields + # Check if primitive has opts field and it contains semantic data + semantic = case Map.get(primitive, :opts) do + nil -> nil + opts when is_list(opts) -> Keyword.get(opts, :semantic) + _ -> nil + end + + if semantic do + element_info = %{ + id: id, + type: primitive.module, + semantic: semantic, + # Extract text content for text primitives + content: extract_content(primitive), + # Store transform for position info if needed + transforms: primitive.transforms + } + Map.put(acc, id, element_info) + else + acc + end + end) + + %{ + graph_key: graph_key, + timestamp: System.system_time(:millisecond), + elements: elements, + # Quick access indices + by_type: group_elements_by_semantic_type(elements) + } + end + + defp extract_content(%{module: Scenic.Primitive.Text, data: text}), do: text + defp extract_content(_), do: nil + + defp group_elements_by_semantic_type(elements) do + elements + |> Enum.reduce(%{}, fn {id, element}, acc -> + if type = get_in(element, [:semantic, :type]) do + Map.update(acc, type, [id], &[id | &1]) + else + acc + end + end) + end end diff --git a/scenic-semantic-handover.md b/scenic-semantic-handover.md new file mode 100644 index 00000000..2022f06d --- /dev/null +++ b/scenic-semantic-handover.md @@ -0,0 +1,445 @@ +# Scenic Semantic Layer Implementation Handover + +## Context and Goal + +We need to add a minimal semantic layer to Scenic that allows test frameworks (particularly spex-driven tests) to query and assert on the meaning/content of GUI elements, not just their visual representation. This is especially important for testing text editors like Quillex/Flamelex where we need to verify buffer contents. + +The current problem: Scenic's ViewPort only stores rendering scripts in ETS, making it impossible to query "what text is in buffer 1?" or "which element is the submit button?" during tests. + +## Core Implementation Strategy + +Add a parallel semantic information system that: +1. Extracts semantic meaning from Graphs during compilation +2. Stores this information alongside rendering scripts +3. Provides simple query APIs for testing + +## Files to Modify + +### 1. `/lib/scenic/view_port.ex` + +**Add semantic table to ViewPort struct:** + +```elixir +defstruct [ + # ... existing fields ... + script_table: nil, + semantic_table: nil, # NEW: Add this field + # ... rest of existing fields ... +] +``` + +**Modify `init/1` to create semantic table:** + +```elixir +def init(opts) do + # ... existing code ... + + # After creating script_table, add: + semantic_table = :ets.new(:semantic_table, [:set, :public]) + + # Update struct initialization to include semantic_table +end +``` + +**Modify `put/4` to build and store semantic info:** + +```elixir +def put(%ViewPort{} = vp, graph_key, %Graph{} = graph, opts \\ []) do + # Existing: compile graph to scripts + scripts = Graph.compile(graph) + + # NEW: Build semantic information + semantic_info = build_semantic_info(graph, graph_key) + + # Store both + :ets.insert(vp.script_table, {graph_key, scripts}) + :ets.insert(vp.semantic_table, {graph_key, semantic_info}) # NEW + + # ... rest of existing implementation ... +end + +# NEW: Add this private function +defp build_semantic_info(graph, graph_key) do + elements = + graph.primitives + |> Enum.reduce(%{}, fn {id, primitive}, acc -> + # Extract semantic data if present + if semantic = get_in(primitive, [:opts, :semantic]) do + element_info = %{ + id: id, + type: primitive.module, + semantic: semantic, + # Extract text content for text primitives + content: extract_content(primitive), + # Store transform for position info if needed + transform: primitive.transforms + } + Map.put(acc, id, element_info) + else + acc + end + end) + + %{ + graph_key: graph_key, + timestamp: System.system_time(:millisecond), + elements: elements, + # Quick access indices + by_type: group_elements_by_semantic_type(elements) + } +end + +defp extract_content(%{module: Scenic.Primitive.Text, data: text}), do: text +defp extract_content(_), do: nil + +defp group_elements_by_semantic_type(elements) do + elements + |> Enum.reduce(%{}, fn {id, element}, acc -> + if type = get_in(element, [:semantic, :type]) do + Map.update(acc, type, [id], &[id | &1]) + else + acc + end + end) +end +``` + +### 2. Create `/lib/scenic/semantic.ex` + +```elixir +defmodule Scenic.Semantic do + @moduledoc """ + Semantic information helpers for Scenic components. + + Provides consistent semantic annotations for testing and accessibility. + """ + + @doc """ + Mark an element as a button. + + ## Examples + |> rect({100, 40}, semantic: Scenic.Semantic.button("Submit")) + """ + def button(label) do + %{type: :button, label: label, role: :button} + end + + @doc """ + Mark an element as an editable text buffer. + + ## Examples + |> text(content, semantic: Scenic.Semantic.text_buffer(buffer_id: 1)) + """ + def text_buffer(opts) do + %{ + type: :text_buffer, + buffer_id: Keyword.fetch!(opts, :buffer_id), + editable: Keyword.get(opts, :editable, true), + role: :textbox + } + end + + @doc """ + Mark an element as a text input field. + """ + def text_input(name, opts \\ []) do + %{ + type: :text_input, + name: name, + value: Keyword.get(opts, :value), + placeholder: Keyword.get(opts, :placeholder), + role: :textbox + } + end + + @doc """ + Mark an element as a menu. + """ + def menu(name, opts \\ []) do + %{ + type: :menu, + name: name, + orientation: Keyword.get(opts, :orientation, :vertical), + role: :menu + } + end + + @doc """ + Mark an element as a menu item. + """ + def menu_item(label, opts \\ []) do + %{ + type: :menu_item, + label: label, + parent_menu: Keyword.get(opts, :parent_menu), + role: :menuitem + } + end + + @doc """ + Generic semantic annotation. + """ + def annotate(type, attrs \\ %{}) do + Map.merge(%{type: type}, attrs) + end +end +``` + +### 3. Create `/lib/scenic/semantic/query.ex` + +```elixir +defmodule Scenic.Semantic.Query do + @moduledoc """ + Query API for semantic information in Scenic ViewPorts. + + Provides testing-friendly functions to find and inspect GUI elements + based on their semantic meaning rather than visual properties. + """ + + @doc """ + Get semantic information for a graph. + """ + def get_semantic_info(viewport, graph_key \\ :main) do + case :ets.lookup(viewport.semantic_table, graph_key) do + [{^graph_key, info}] -> {:ok, info} + [] -> {:error, :no_semantic_info} + end + end + + @doc """ + Find all elements of a specific semantic type. + + ## Examples + Query.find_by_type(viewport, :button) + Query.find_by_type(viewport, :text_buffer) + """ + def find_by_type(viewport, type, graph_key \\ :main) do + with {:ok, info} <- get_semantic_info(viewport, graph_key) do + ids = get_in(info, [:by_type, type]) || [] + elements = Enum.map(ids, &Map.get(info.elements, &1)) + {:ok, elements} + end + end + + @doc """ + Find a single element by semantic type and additional filter. + + ## Examples + Query.find_one(viewport, :text_buffer, fn elem -> + elem.semantic.buffer_id == 1 + end) + """ + def find_one(viewport, type, filter_fn, graph_key \\ :main) do + with {:ok, elements} <- find_by_type(viewport, type, graph_key) do + case Enum.find(elements, filter_fn) do + nil -> {:error, :not_found} + element -> {:ok, element} + end + end + end + + @doc """ + Get text content from a text buffer by buffer_id. + """ + def get_buffer_text(viewport, buffer_id, graph_key \\ :main) do + with {:ok, buffer} <- find_one(viewport, :text_buffer, fn elem -> + elem.semantic.buffer_id == buffer_id + end, graph_key) do + {:ok, buffer.content || ""} + end + end + + @doc """ + Find all buttons in the viewport. + """ + def get_buttons(viewport, graph_key \\ :main) do + find_by_type(viewport, :button, graph_key) + end + + @doc """ + Find button by label. + """ + def get_button_by_label(viewport, label, graph_key \\ :main) do + find_one(viewport, :button, fn elem -> + elem.semantic.label == label + end, graph_key) + end + + @doc """ + Get all editable text content. + """ + def get_editable_content(viewport, graph_key \\ :main) do + with {:ok, info} <- get_semantic_info(viewport, graph_key) do + editable = + info.elements + |> Map.values() + |> Enum.filter(fn elem -> + get_in(elem, [:semantic, :editable]) == true + end) + |> Enum.map(fn elem -> + %{ + id: elem.id, + type: elem.semantic.type, + content: elem.content, + buffer_id: elem.semantic[:buffer_id] + } + end) + {:ok, editable} + end + end + + @doc """ + Debug helper - print all semantic elements. + """ + def inspect_semantic_tree(viewport, graph_key \\ :main) do + with {:ok, info} <- get_semantic_info(viewport, graph_key) do + IO.puts("=== Semantic Tree for #{graph_key} ===") + IO.puts("Total elements: #{map_size(info.elements)}") + IO.puts("\nBy type:") + + Enum.each(info.by_type, fn {type, ids} -> + IO.puts(" #{type}: #{length(ids)} elements") + Enum.each(ids, fn id -> + elem = Map.get(info.elements, id) + IO.puts(" - #{id}: #{inspect(elem.semantic)}") + end) + end) + + :ok + end + end +end +``` + +### 4. Update `/lib/scenic/graph.ex` + +We need to ensure semantic options are preserved through graph operations. Look for the `add/4` function and similar primitive-adding functions. Make sure `:semantic` is included in allowed options. + +This might already work if Scenic passes through all options, but verify that semantic options aren't stripped during graph compilation. + +## Testing the Implementation + +### 1. Create a test file `test/scenic/semantic_test.exs`: + +```elixir +defmodule Scenic.SemanticTest do + use ExUnit.Case + alias Scenic.{ViewPort, Graph, Semantic} + alias Scenic.Semantic.Query + + setup do + {:ok, viewport} = ViewPort.start_link(name: :semantic_test_vp) + {:ok, viewport: viewport} + end + + test "semantic info is stored when graph is put", %{viewport: viewport} do + graph = + Graph.build() + |> Scenic.Primitives.rect({100, 40}, semantic: Semantic.button("Click me")) + |> Scenic.Primitives.text("Click me", semantic: %{type: :label, for: :button}) + + ViewPort.put(viewport, :test_graph, graph) + + assert {:ok, info} = Query.get_semantic_info(viewport, :test_graph) + assert map_size(info.elements) == 2 + assert info.by_type.button != nil + end + + test "can query buttons by label", %{viewport: viewport} do + graph = + Graph.build() + |> Scenic.Primitives.rect({100, 40}, semantic: Semantic.button("Submit")) + |> Scenic.Primitives.rect({100, 40}, semantic: Semantic.button("Cancel")) + + ViewPort.put(viewport, graph) + + assert {:ok, submit_btn} = Query.get_button_by_label(viewport, "Submit") + assert submit_btn.semantic.label == "Submit" + + assert {:ok, buttons} = Query.get_buttons(viewport) + assert length(buttons) == 2 + end + + test "can query text buffer content", %{viewport: viewport} do + buffer_content = "Hello, World!" + + graph = + Graph.build() + |> Scenic.Primitives.text(buffer_content, + semantic: Semantic.text_buffer(buffer_id: 1)) + + ViewPort.put(viewport, graph) + + assert {:ok, ^buffer_content} = Query.get_buffer_text(viewport, 1) + end +end +``` + +### 2. Integration test with actual Flamelex/Quillex component: + +```elixir +# In your text editor test +test "can read buffer content through semantic layer" do + # Setup your editor component + {:ok, scene} = MyEditor.start_link(viewport: viewport) + + # Type some text (using your existing input simulation) + Scene.cast(scene, {:input, "Hello semantic world"}) + + # Query through semantic API + assert {:ok, "Hello semantic world"} = Query.get_buffer_text(viewport, 1) +end +``` + +## Migration Guide for Existing Components + +To add semantic information to existing Quillex/Flamelex components: + +```elixir +# Before: +@graph Graph.build() + |> text(buffer.content, id: :buffer_text) + +# After: +@graph Graph.build() + |> text(buffer.content, + id: :buffer_text, + semantic: Semantic.text_buffer(buffer_id: buffer.id)) + +# For buttons: +# Before: +|> rect(button_size, id: :save_button) +|> text("Save", id: :save_label) + +# After: +|> rect(button_size, id: :save_button, semantic: Semantic.button("Save")) +|> text("Save", id: :save_label) +``` + +## Success Criteria + +1. Can query buffer text content in tests without parsing viewport scripts +2. Can find and identify buttons by their labels +3. Zero performance impact on non-semantic components +4. Existing Scenic apps continue working without modification +5. Simple API that's intuitive for test writers + +## Next Steps After Basic Implementation + +Once the basic implementation is working: + +1. Add semantic info to Quillex BufferPane component +2. Update comprehensive_text_editing_spex.exs to use semantic queries +3. Document common semantic patterns +4. Consider adding semantic info to ScriptInspector for backwards compatibility + +## Implementation Order + +1. Start with ViewPort.ex changes +2. Create Semantic module with helper functions +3. Create Query module for testing +4. Write tests to verify it works +5. Update one Quillex component as proof of concept +6. Update spex tests to use new Query API + +The goal is a working proof of concept that can query buffer text within 1-2 days, then iterate based on real usage. \ No newline at end of file diff --git a/test/scenic/semantic_test.exs b/test/scenic/semantic_test.exs new file mode 100644 index 00000000..86f3c160 --- /dev/null +++ b/test/scenic/semantic_test.exs @@ -0,0 +1,75 @@ +defmodule Scenic.SemanticTest do + use ExUnit.Case + alias Scenic.{ViewPort, Graph, Semantic} + alias Scenic.Semantic.Query + + setup_all do + # Start the Scenic supervisor if not already started + case Process.whereis(:scenic) do + nil -> + {:ok, _} = Scenic.start_link([]) + _pid -> + :ok + end + :ok + end + + setup do + # Generate a unique name for each test + name = :"semantic_test_vp_#{System.unique_integer([:positive])}" + + {:ok, viewport} = ViewPort.start([ + name: name, + size: {800, 600}, + default_scene: {Scenic.Scene, nil} + ]) + + on_exit(fn -> + # Clean up the viewport + ViewPort.stop(viewport) + end) + + {:ok, viewport: viewport} + end + + test "semantic info is stored when graph is put", %{viewport: viewport} do + graph = + Graph.build() + |> Scenic.Primitives.rect({100, 40}, semantic: Semantic.button("Click me")) + |> Scenic.Primitives.text("Click me", semantic: %{type: :label, for: :button}) + + ViewPort.put_graph(viewport, :test_graph, graph) + + assert {:ok, info} = Query.get_semantic_info(viewport, :test_graph) + assert map_size(info.elements) == 2 + assert info.by_type.button != nil + end + + test "can query buttons by label", %{viewport: viewport} do + graph = + Graph.build() + |> Scenic.Primitives.rect({100, 40}, semantic: Semantic.button("Submit")) + |> Scenic.Primitives.rect({100, 40}, semantic: Semantic.button("Cancel")) + + ViewPort.put_graph(viewport, :test_graph, graph) + + assert {:ok, submit_btn} = Query.get_button_by_label(viewport, "Submit", :test_graph) + assert submit_btn.semantic.label == "Submit" + + assert {:ok, buttons} = Query.get_buttons(viewport, :test_graph) + assert length(buttons) == 2 + end + + test "can query text buffer content", %{viewport: viewport} do + buffer_content = "Hello, World!" + + graph = + Graph.build() + |> Scenic.Primitives.text(buffer_content, + semantic: Semantic.text_buffer(buffer_id: 1)) + + ViewPort.put_graph(viewport, :test_graph, graph) + + assert {:ok, ^buffer_content} = Query.get_buffer_text(viewport, 1, :test_graph) + end +end \ No newline at end of file From f627c8aebb79a629952bc2b70fc79b8c60921aa7 Mon Sep 17 00:00:00 2001 From: JediLuke <1879634+JediLuke@users.noreply.github.com> Date: Sun, 13 Jul 2025 22:14:26 -0500 Subject: [PATCH 03/11] looking great but WIp! --- lib/scenic.ex | 75 +++++-- lib/scenic/dev_tools_inspector.ex | 341 ++++++++++++++++++++++++++++++ lib/scenic/view_port.ex | 31 +-- 3 files changed, 410 insertions(+), 37 deletions(-) create mode 100644 lib/scenic/dev_tools_inspector.ex diff --git a/lib/scenic.ex b/lib/scenic.ex index 248ea303..c030af46 100644 --- a/lib/scenic.ex +++ b/lib/scenic.ex @@ -95,10 +95,8 @@ defmodule Scenic do ## Examples - iex> Scenic.semantic() - === Semantic Tree for :main === - Total elements: 3 - ... + Scenic.semantic() + # Prints semantic tree information to console """ defdelegate semantic(viewport_name \\ :main_viewport, graph_key \\ :main), to: Scenic.DevTools @@ -110,10 +108,8 @@ defmodule Scenic do ## Examples - iex> Scenic.buffers() - Text Buffers: - [1] "Hello, World!" - ... + Scenic.buffers() + # Prints text buffer information to console """ defdelegate buffers(viewport_name \\ :main_viewport, graph_key \\ :main), to: Scenic.DevTools @@ -125,9 +121,8 @@ defmodule Scenic do ## Examples - iex> Scenic.buffer(1) - Buffer 1: - Hello, World! + Scenic.buffer(1) + # Prints content of buffer 1 to console """ defdelegate buffer(buffer_id, viewport_name \\ :main_viewport, graph_key \\ :main), to: Scenic.DevTools @@ -139,10 +134,8 @@ defmodule Scenic do ## Examples - iex> Scenic.buttons() - Buttons: - - "Save" (id: :save_btn) - ... + Scenic.buttons() + # Prints button information to console """ defdelegate buttons(viewport_name \\ :main_viewport, graph_key \\ :main), to: Scenic.DevTools @@ -154,9 +147,8 @@ defmodule Scenic do ## Examples - iex> Scenic.find(:menu) - Found 1 menu element(s): - - :main_menu: %{type: :menu, name: "File"} + Scenic.find(:menu) + # Finds and prints menu elements """ defdelegate find(type, viewport_name \\ :main_viewport, graph_key \\ :main), to: Scenic.DevTools @@ -168,11 +160,50 @@ defmodule Scenic do ## Examples - iex> Scenic.types() - Semantic types in use: - - button (2 elements) - - text_buffer (1 element) + Scenic.types() + # Lists all semantic types in use """ defdelegate types(viewport_name \\ :main_viewport, graph_key \\ :main), to: Scenic.DevTools + + # High-level inspection tools + + @doc """ + Inspect your entire Scenic application like browser dev tools. + + Shows a hierarchical view starting from the ViewPort with all scenes, + graphs, and semantic annotations. Perfect for understanding your app structure. + + ## Examples + + Scenic.inspect_app() + # Shows hierarchical view of your entire app + """ + defdelegate inspect_app(viewport_name \\ :main_viewport), + to: Scenic.DevToolsInspector + + @doc """ + Show just the semantic content in your app. + + Like a simplified view showing "what semantic stuff is in my app?" + Perfect for beginners. + + ## Examples + + Scenic.show_semantic() + # Shows all semantic content grouped by type + """ + defdelegate show_semantic(viewport_name \\ :main_viewport), + to: Scenic.DevToolsInspector, as: :show_semantic_content + + @doc """ + Inspect a specific graph in detail. + + ## Examples + + Scenic.inspect_graph("ABC123...") + # Shows detailed view of one graph + """ + defdelegate inspect_graph(graph_key, viewport_name \\ :main_viewport), + to: Scenic.DevToolsInspector end diff --git a/lib/scenic/dev_tools_inspector.ex b/lib/scenic/dev_tools_inspector.ex new file mode 100644 index 00000000..7aad8724 --- /dev/null +++ b/lib/scenic/dev_tools_inspector.ex @@ -0,0 +1,341 @@ +defmodule Scenic.DevToolsInspector do + @moduledoc """ + High-level Scenic application inspector - like browser dev tools for Scenic. + + Provides a hierarchical view of your Scenic application starting from the root, + making it easy to understand the scene structure and semantic annotations. + """ + + alias Scenic.ViewPort + + @doc """ + Inspect the entire Scenic application hierarchy. + + Shows a tree-like view starting from the ViewPort, listing all scenes/graphs + and highlighting which ones have semantic annotations. + + ## Examples + + iex> Scenic.DevToolsInspector.inspect_app() + === Scenic Application Inspector === + ViewPort: :main_viewport (1440x855) + + 📊 Total Graphs: 8 + 🏷️ Graphs with Semantic Data: 2 + + 📋 All Graphs: + ├── 🏷️ "ABC123..." (5 semantic elements) + │ ├── 📝 text_buffer: buffer_id "uuid1" + │ ├── 🔘 button: "Save" + │ └── 📂 menu: "File" + ├── ⚪ "DEF456..." (no semantic data) + ├── 🏷️ "GHI789..." (1 semantic element) + │ └── 📝 text_buffer: buffer_id "uuid2" + └── ⚪ "JKL012..." (no semantic data) + """ + def inspect_app(viewport_name \\ :main_viewport) do + with {:ok, viewport} <- get_viewport(viewport_name) do + IO.puts("=== Scenic Application Inspector ===") + IO.puts("ViewPort: #{inspect(viewport_name)} (#{format_size(viewport.size)})") + IO.puts("") + + # Get all semantic data + all_entries = :ets.tab2list(viewport.semantic_table) + + graphs_with_semantic = Enum.filter(all_entries, fn {_key, data} -> + map_size(data.elements) > 0 + end) + + total_graphs = length(all_entries) + semantic_graphs = length(graphs_with_semantic) + + IO.puts("📊 Total Graphs: #{total_graphs}") + IO.puts("🏷️ Graphs with Semantic Data: #{semantic_graphs}") + IO.puts("") + + if semantic_graphs == 0 do + IO.puts("ℹ️ No semantic annotations found. Add semantic metadata to your components:") + IO.puts(" |> text(\"Hello\", semantic: Semantic.text_buffer(buffer_id: 1))") + IO.puts(" |> rect({100, 40}, semantic: Semantic.button(\"Save\"))") + else + IO.puts("📋 All Graphs:") + render_graph_tree(all_entries) + end + + IO.puts("") + IO.puts("💡 Use inspect_graph(\"graph_id\") to see details of a specific graph") + + :ok + else + error -> + IO.puts("Error: #{inspect(error)}") + :error + end + end + + @doc """ + Inspect a specific graph in detail. + + Shows the full semantic structure of a single graph, like zooming in + on one component in the browser dev tools. + + ## Examples + + iex> Scenic.DevToolsInspector.inspect_graph("ABC123...") + === Graph Details: ABC123... === + + 🏷️ Semantic Elements: 5 + 📅 Last Updated: 2024-01-15 14:30:22 + + 📝 text_buffer (1): + └── Element #7: %{buffer_id: "uuid1", editable: true, role: :textbox} + Content: "Hello, World!" + + 🔘 button (2): + └── Element #12: %{label: "Save"} + └── Element #15: %{label: "Cancel"} + """ + def inspect_graph(graph_key, viewport_name \\ :main_viewport) do + with {:ok, viewport} <- get_viewport(viewport_name) do + case :ets.lookup(viewport.semantic_table, graph_key) do + [{^graph_key, data}] -> + IO.puts("=== Graph Details: #{String.slice(graph_key, 0, 8)}... ===") + IO.puts("") + + element_count = map_size(data.elements) + timestamp = format_timestamp(data.timestamp) + + IO.puts("🏷️ Semantic Elements: #{element_count}") + IO.puts("📅 Last Updated: #{timestamp}") + IO.puts("") + + if element_count == 0 do + IO.puts("⚪ No semantic elements in this graph") + else + render_semantic_details(data) + end + + :ok + + [] -> + IO.puts("❌ Graph not found: #{graph_key}") + + # Show available graphs + all_entries = :ets.tab2list(viewport.semantic_table) + IO.puts("\n📋 Available graphs:") + Enum.each(all_entries, fn {key, _data} -> + short_key = String.slice(key, 0, 8) <> "..." + IO.puts(" #{short_key}") + end) + + :error + end + else + error -> + IO.puts("Error: #{inspect(error)}") + :error + end + end + + @doc """ + Show just the semantic content - like a simplified view. + + Perfect for beginners who just want to see "what semantic stuff is in my app?" + + ## Examples + + iex> Scenic.DevToolsInspector.show_semantic_content() + === Semantic Content in Your App === + + 📝 Text Buffers (2): + • Buffer "uuid1": "Hello, World!" + • Buffer "uuid2": "def hello do..." + + 🔘 Buttons (3): + • "Save" + • "Cancel" + • "Submit" + + 📂 Menus (1): + • "File" menu + """ + def show_semantic_content(viewport_name \\ :main_viewport) do + with {:ok, viewport} <- get_viewport(viewport_name) do + IO.puts("=== Semantic Content in Your App ===") + IO.puts("") + + # Collect all semantic elements across all graphs + all_entries = :ets.tab2list(viewport.semantic_table) + + all_elements = Enum.flat_map(all_entries, fn {_key, data} -> + Map.values(data.elements) + end) + + if all_elements == [] do + IO.puts("🚫 No semantic content found") + IO.puts("") + IO.puts("💡 To add semantic annotations to your components:") + IO.puts(" |> text(\"Hello\", semantic: Semantic.text_buffer(buffer_id: 1))") + IO.puts(" |> rect({100, 40}, semantic: Semantic.button(\"Save\"))") + else + # Group by semantic type + by_type = Enum.group_by(all_elements, fn elem -> + elem.semantic.type + end) + + # Show each type + Enum.each(by_type, fn {type, elements} -> + count = length(elements) + icon = get_type_icon(type) + type_name = String.capitalize(to_string(type)) <> "s" + + IO.puts("#{icon} #{type_name} (#{count}):") + + Enum.each(elements, fn elem -> + description = format_element_description(elem) + IO.puts(" • #{description}") + end) + + IO.puts("") + end) + end + + :ok + else + error -> + IO.puts("Error: #{inspect(error)}") + :error + end + end + + # Private helpers + + defp render_graph_tree(all_entries) do + {graphs_with_semantic, graphs_without} = Enum.split_with(all_entries, fn {_key, data} -> + map_size(data.elements) > 0 + end) + + # Show graphs with semantic data first + Enum.with_index(graphs_with_semantic, fn {key, data}, index -> + is_last_semantic = index == length(graphs_with_semantic) - 1 + has_more_graphs = length(graphs_without) > 0 + + connector = if is_last_semantic and not has_more_graphs, do: "└──", else: "├──" + + short_key = String.slice(key, 0, 8) <> "..." + element_count = map_size(data.elements) + + IO.puts("#{connector} 🏷️ \"#{short_key}\" (#{element_count} semantic elements)") + + # Show a preview of elements + data.elements + |> Map.values() + |> Enum.take(3) + |> Enum.with_index() + |> Enum.each(fn {elem, elem_index} -> + is_last_elem = elem_index == min(2, element_count - 1) + elem_connector = if is_last_semantic and not has_more_graphs and is_last_elem, do: " └──", else: "│ ├──" + + icon = get_type_icon(elem.semantic.type) + description = format_element_brief(elem) + IO.puts("#{elem_connector} #{icon} #{description}") + end) + + if element_count > 3 do + more_connector = if is_last_semantic and not has_more_graphs, do: " └──", else: "│ └──" + IO.puts("#{more_connector} ... and #{element_count - 3} more") + end + end) + + # Show graphs without semantic data + Enum.with_index(graphs_without, fn {key, _data}, index -> + is_last = index == length(graphs_without) - 1 + connector = if is_last, do: "└──", else: "├──" + + short_key = String.slice(key, 0, 8) <> "..." + IO.puts("#{connector} ⚪ \"#{short_key}\" (no semantic data)") + end) + end + + defp render_semantic_details(data) do + Enum.each(data.by_type, fn {type, element_ids} -> + icon = get_type_icon(type) + type_name = String.capitalize(to_string(type)) + count = length(element_ids) + + IO.puts("#{icon} #{type_name} (#{count}):") + + Enum.each(element_ids, fn id -> + elem = Map.get(data.elements, id) + description = format_element_description(elem) + IO.puts(" └── Element ##{id}: #{description}") + + # Show content if it's a text element + if elem.content && String.trim(elem.content) != "" do + content_preview = String.slice(elem.content, 0, 50) + content_preview = if String.length(elem.content) > 50, do: content_preview <> "...", else: content_preview + IO.puts(" Content: #{inspect(content_preview)}") + end + end) + + IO.puts("") + end) + end + + defp format_element_brief(elem) do + case elem.semantic.type do + :text_buffer -> "text_buffer: buffer_id \"#{String.slice(elem.semantic.buffer_id, 0, 8)}...\"" + :button -> "button: \"#{elem.semantic.label}\"" + :menu -> "menu: \"#{elem.semantic.name}\"" + :text_input -> "text_input: \"#{elem.semantic.name}\"" + other -> "#{other}: #{inspect(elem.semantic)}" + end + end + + defp format_element_description(elem) do + case elem.semantic.type do + :text_buffer -> + id = String.slice(elem.semantic.buffer_id, 0, 8) <> "..." + preview = if elem.content && String.trim(elem.content) != "" do + " - \"#{String.slice(elem.content, 0, 20)}...\"" + else + " (empty)" + end + "Buffer #{id}#{preview}" + + :button -> "\"#{elem.semantic.label}\"" + :menu -> "\"#{elem.semantic.name}\" menu" + :text_input -> "\"#{elem.semantic.name}\" input" + _other -> "#{inspect(elem.semantic)}" + end + end + + defp get_type_icon(type) do + case type do + :text_buffer -> "📝" + :button -> "🔘" + :menu -> "📂" + :text_input -> "📝" + _ -> "🏷️" + end + end + + defp format_size({width, height}), do: "#{width}x#{height}" + defp format_size(_), do: "unknown" + + defp format_timestamp(timestamp) when is_integer(timestamp) do + # Convert from milliseconds since Unix epoch + datetime = DateTime.from_unix!(timestamp, :millisecond) + Calendar.strftime(datetime, "%Y-%m-%d %H:%M:%S") + end + defp format_timestamp(_), do: "unknown" + + defp get_viewport(viewport) when is_struct(viewport, ViewPort), do: {:ok, viewport} + defp get_viewport(name) when is_atom(name) do + case Process.whereis(name) do + nil -> {:error, "ViewPort #{inspect(name)} not found"} + pid -> ViewPort.info(pid) + end + end + defp get_viewport(pid) when is_pid(pid), do: ViewPort.info(pid) +end \ No newline at end of file diff --git a/lib/scenic/view_port.ex b/lib/scenic/view_port.ex index 3567e48a..d645e6ec 100644 --- a/lib/scenic/view_port.ex +++ b/lib/scenic/view_port.ex @@ -674,11 +674,11 @@ defmodule Scenic.ViewPort do ## Examples - iex> ViewPort.get_semantic(viewport) - {:ok, %{elements: %{...}, by_type: %{...}}} + ViewPort.get_semantic(viewport) + # => {:ok, %{elements: %{...}, by_type: %{...}}} - iex> ViewPort.get_semantic(viewport, :specific_graph) - {:ok, %{...}} + ViewPort.get_semantic(viewport, :specific_graph) + # => {:ok, %{...}} """ @spec get_semantic(viewport :: ViewPort.t(), graph_key :: any) :: {:ok, map} | {:error, :no_semantic_info} @@ -695,17 +695,18 @@ defmodule Scenic.ViewPort do ## Examples - iex> ViewPort.inspect_semantic(viewport) - === Semantic Tree for :main === - Total elements: 5 - - By type: - button: 2 elements - - :submit_btn: %{type: :button, label: "Submit"} - - :cancel_btn: %{type: :button, label: "Cancel"} - text_buffer: 1 element - - :buffer_1: %{type: :text_buffer, buffer_id: 1} - :ok + ViewPort.inspect_semantic(viewport) + # Prints: + # === Semantic Tree for :main === + # Total elements: 5 + # + # By type: + # button: 2 elements + # - :submit_btn: %{type: :button, label: "Submit"} + # - :cancel_btn: %{type: :button, label: "Cancel"} + # text_buffer: 1 element + # - :buffer_1: %{type: :text_buffer, buffer_id: 1} + # => :ok """ @spec inspect_semantic(viewport :: ViewPort.t(), graph_key :: any) :: :ok def inspect_semantic(%ViewPort{} = viewport, graph_key \\ :main) do From ccc5e71942e3fe43de391e59281087df20039dd4 Mon Sep 17 00:00:00 2001 From: JediLuke <1879634+JediLuke@users.noreply.github.com> Date: Sun, 13 Jul 2025 22:21:18 -0500 Subject: [PATCH 04/11] merge into one --- lib/scenic.ex | 121 -------- lib/scenic/dev_tools.ex | 475 +++++++++++++++++++++++++++++- lib/scenic/dev_tools_enhanced.ex | 143 --------- lib/scenic/dev_tools_inspector.ex | 341 --------------------- 4 files changed, 464 insertions(+), 616 deletions(-) delete mode 100644 lib/scenic/dev_tools_enhanced.ex delete mode 100644 lib/scenic/dev_tools_inspector.ex diff --git a/lib/scenic.ex b/lib/scenic.ex index c030af46..57a6521a 100644 --- a/lib/scenic.ex +++ b/lib/scenic.ex @@ -85,125 +85,4 @@ defmodule Scenic do |> Supervisor.init(strategy: :one_for_one) end - # ========================================================================= - # Developer Tools - Convenience delegates - - @doc """ - Display semantic information for the current viewport. - - Convenience function that delegates to `Scenic.DevTools.semantic/2`. - - ## Examples - - Scenic.semantic() - # Prints semantic tree information to console - """ - defdelegate semantic(viewport_name \\ :main_viewport, graph_key \\ :main), - to: Scenic.DevTools - - @doc """ - Show all text buffers and their content. - - Convenience function that delegates to `Scenic.DevTools.buffers/2`. - - ## Examples - - Scenic.buffers() - # Prints text buffer information to console - """ - defdelegate buffers(viewport_name \\ :main_viewport, graph_key \\ :main), - to: Scenic.DevTools - - @doc """ - Show content of a specific buffer. - - Convenience function that delegates to `Scenic.DevTools.buffer/3`. - - ## Examples - - Scenic.buffer(1) - # Prints content of buffer 1 to console - """ - defdelegate buffer(buffer_id, viewport_name \\ :main_viewport, graph_key \\ :main), - to: Scenic.DevTools - - @doc """ - Show all buttons in the viewport. - - Convenience function that delegates to `Scenic.DevTools.buttons/2`. - - ## Examples - - Scenic.buttons() - # Prints button information to console - """ - defdelegate buttons(viewport_name \\ :main_viewport, graph_key \\ :main), - to: Scenic.DevTools - - @doc """ - Find elements by semantic type. - - Convenience function that delegates to `Scenic.DevTools.find/3`. - - ## Examples - - Scenic.find(:menu) - # Finds and prints menu elements - """ - defdelegate find(type, viewport_name \\ :main_viewport, graph_key \\ :main), - to: Scenic.DevTools - - @doc """ - List all semantic types in use. - - Convenience function that delegates to `Scenic.DevTools.types/2`. - - ## Examples - - Scenic.types() - # Lists all semantic types in use - """ - defdelegate types(viewport_name \\ :main_viewport, graph_key \\ :main), - to: Scenic.DevTools - - # High-level inspection tools - - @doc """ - Inspect your entire Scenic application like browser dev tools. - - Shows a hierarchical view starting from the ViewPort with all scenes, - graphs, and semantic annotations. Perfect for understanding your app structure. - - ## Examples - - Scenic.inspect_app() - # Shows hierarchical view of your entire app - """ - defdelegate inspect_app(viewport_name \\ :main_viewport), - to: Scenic.DevToolsInspector - - @doc """ - Show just the semantic content in your app. - - Like a simplified view showing "what semantic stuff is in my app?" - Perfect for beginners. - - ## Examples - - Scenic.show_semantic() - # Shows all semantic content grouped by type - """ - defdelegate show_semantic(viewport_name \\ :main_viewport), - to: Scenic.DevToolsInspector, as: :show_semantic_content - - @doc """ - Inspect a specific graph in detail. - - ## Examples - - Scenic.inspect_graph("ABC123...") - # Shows detailed view of one graph - """ - defdelegate inspect_graph(graph_key, viewport_name \\ :main_viewport), - to: Scenic.DevToolsInspector end diff --git a/lib/scenic/dev_tools.ex b/lib/scenic/dev_tools.ex index f980accf..4a9b8a84 100644 --- a/lib/scenic/dev_tools.ex +++ b/lib/scenic/dev_tools.ex @@ -1,21 +1,29 @@ defmodule Scenic.DevTools do @moduledoc """ - Developer tools for inspecting Scenic applications during development. + Comprehensive developer tools for inspecting Scenic applications during development. - Import this module in your IEx session for easy access to semantic - inspection and debugging tools. + This module combines basic semantic inspection, UUID-aware enhanced tools, + and high-level "browser dev tools" style inspection into a single namespace. ## Usage in IEx iex> import Scenic.DevTools + + # High-level inspection (recommended) + iex> inspect_app() # Browser dev tools style view + iex> show_semantic() # Simple semantic content overview + + # Detailed queries iex> semantic() # Show semantic info for default viewport - iex> semantic(:my_viewport) # Show semantic info for named viewport iex> buffers() # Show all text buffers iex> buttons() # Show all buttons + + # Enhanced tools for UUID graphs + iex> semantic_all() # All semantic data across graphs + iex> buffers_all() # All buffers across graphs """ - alias Scenic.{ViewPort, Semantic} - alias Scenic.Semantic.Query + alias Scenic.ViewPort @doc """ Display semantic information for the current viewport. @@ -55,7 +63,7 @@ defmodule Scenic.DevTools do """ def buffers(viewport_name \\ :main_viewport, graph_key \\ :main) do with {:ok, viewport} <- get_viewport(viewport_name), - {:ok, buffers} <- Query.find_by_type(viewport, :text_buffer, graph_key) do + {:ok, buffers} <- Scenic.Semantic.Query.find_by_type(viewport, :text_buffer, graph_key) do IO.puts("Text Buffers:") Enum.each(buffers, fn buffer -> content = buffer.content || "" @@ -86,7 +94,7 @@ defmodule Scenic.DevTools do """ def buffer(buffer_id, viewport_name \\ :main_viewport, graph_key \\ :main) do with {:ok, viewport} <- get_viewport(viewport_name), - {:ok, content} <- Query.get_buffer_text(viewport, buffer_id, graph_key) do + {:ok, content} <- Scenic.Semantic.Query.get_buffer_text(viewport, buffer_id, graph_key) do IO.puts("Buffer #{buffer_id}:") IO.puts(content) :ok @@ -109,7 +117,7 @@ defmodule Scenic.DevTools do """ def buttons(viewport_name \\ :main_viewport, graph_key \\ :main) do with {:ok, viewport} <- get_viewport(viewport_name), - {:ok, buttons} <- Query.get_buttons(viewport, graph_key) do + {:ok, buttons} <- Scenic.Semantic.Query.get_buttons(viewport, graph_key) do IO.puts("Buttons:") Enum.each(buttons, fn button -> label = button.semantic.label @@ -138,7 +146,7 @@ defmodule Scenic.DevTools do """ def find(type, viewport_name \\ :main_viewport, graph_key \\ :main) do with {:ok, viewport} <- get_viewport(viewport_name), - {:ok, elements} <- Query.find_by_type(viewport, type, graph_key) do + {:ok, elements} <- Scenic.Semantic.Query.find_by_type(viewport, type, graph_key) do IO.puts("Found #{length(elements)} #{type} element(s):") Enum.each(elements, fn elem -> IO.puts("- #{inspect(elem.id)}: #{inspect(elem.semantic)}") @@ -205,7 +213,452 @@ defmodule Scenic.DevTools do end end - # Private helpers + # ============================================================================= + # Enhanced Tools (UUID-aware) + # ============================================================================= + + @doc """ + Display all semantic information across all graphs. + + This function works with UUID graph keys and shows semantic data + from all graphs in the viewport. + """ + def semantic_all(viewport_name \\ :main_viewport) do + with {:ok, viewport} <- get_viewport(viewport_name) do + entries = :ets.tab2list(viewport.semantic_table) + + # Filter to only entries with semantic data + semantic_entries = Enum.filter(entries, fn {_key, data} -> + map_size(data.elements) > 0 + end) + + if semantic_entries == [] do + IO.puts("No semantic information found in any graph") + else + IO.puts("=== Semantic Information Across All Graphs ===") + IO.puts("Found #{length(semantic_entries)} graphs with semantic data\n") + + Enum.each(semantic_entries, fn {graph_key, data} -> + IO.puts("Graph: #{graph_key}") + IO.puts("Elements: #{map_size(data.elements)}") + + if map_size(data.by_type) > 0 do + IO.puts("By type:") + Enum.each(data.by_type, fn {type, ids} -> + IO.puts(" #{type}: #{length(ids)} element(s)") + + # Show details for each element + Enum.each(ids, fn id -> + elem = Map.get(data.elements, id) + case type do + :text_buffer -> + IO.puts(" - Buffer #{elem.semantic.buffer_id}") + content_preview = String.slice(elem.content || "", 0, 50) + if content_preview != "", do: IO.puts(" Content: #{inspect(content_preview)}") + + :button -> + IO.puts(" - Button: #{elem.semantic.label}") + + _ -> + IO.puts(" - #{inspect(elem.semantic)}") + end + end) + end) + end + IO.puts("") + end) + end + :ok + end + end + + @doc """ + Show all text buffers across all graphs. + """ + def buffers_all(viewport_name \\ :main_viewport) do + with {:ok, viewport} <- get_viewport(viewport_name) do + entries = :ets.tab2list(viewport.semantic_table) + + # Find all text buffers + all_buffers = Enum.flat_map(entries, fn {graph_key, data} -> + buffer_ids = Map.get(data.by_type, :text_buffer, []) + + Enum.map(buffer_ids, fn id -> + elem = Map.get(data.elements, id) + %{ + graph_key: graph_key, + buffer_id: elem.semantic.buffer_id, + content: elem.content || "", + semantic: elem.semantic + } + end) + end) + + if all_buffers == [] do + IO.puts("No text buffers found") + else + IO.puts("Text Buffers:") + Enum.each(all_buffers, fn buffer -> + content_preview = String.slice(buffer.content, 0, 60) + content_preview = if String.length(buffer.content) > 60, do: content_preview <> "...", else: content_preview + + IO.puts("\n[Buffer: #{buffer.buffer_id}]") + IO.puts("Graph: #{buffer.graph_key}") + IO.puts("Content: #{inspect(content_preview)}") + end) + end + :ok + end + end + + @doc """ + Get content of a specific buffer by UUID. + """ + def buffer_by_uuid(buffer_uuid, viewport_name \\ :main_viewport) do + with {:ok, viewport} <- get_viewport(viewport_name) do + entries = :ets.tab2list(viewport.semantic_table) + + # Search for the buffer + result = Enum.find_value(entries, fn {_graph_key, data} -> + buffer_ids = Map.get(data.by_type, :text_buffer, []) + + Enum.find_value(buffer_ids, fn id -> + elem = Map.get(data.elements, id) + if elem.semantic.buffer_id == buffer_uuid do + elem.content || "" + end + end) + end) + + case result do + nil -> + IO.puts("Buffer #{buffer_uuid} not found") + :error + content -> + IO.puts("Buffer #{buffer_uuid}:") + IO.puts(content) + :ok + end + end + end + + # ============================================================================= + # High-Level Inspector (Browser Dev Tools Style) + # ============================================================================= + + @doc """ + Inspect your entire Scenic application like browser dev tools. + + Shows a hierarchical view starting from the ViewPort with all scenes, + graphs, and semantic annotations. Perfect for understanding your app structure. + + ## Examples + + iex> inspect_app() + === Scenic Application Inspector === + ViewPort: :main_viewport (1440x855) + + 📊 Total Graphs: 8 + 🏷️ Graphs with Semantic Data: 2 + + 📋 All Graphs: + ├── 🏷️ "ABC123..." (5 semantic elements) + │ ├── 📝 text_buffer: buffer_id "uuid1" + │ ├── 🔘 button: "Save" + │ └── 📂 menu: "File" + ├── ⚪ "DEF456..." (no semantic data) + """ + def inspect_app(viewport_name \\ :main_viewport) do + with {:ok, viewport} <- get_viewport(viewport_name) do + IO.puts("=== Scenic Application Inspector ===") + IO.puts("ViewPort: #{inspect(viewport_name)} (#{format_size(viewport.size)})") + IO.puts("") + + # Get all semantic data + all_entries = :ets.tab2list(viewport.semantic_table) + + graphs_with_semantic = Enum.filter(all_entries, fn {_key, data} -> + map_size(data.elements) > 0 + end) + + total_graphs = length(all_entries) + semantic_graphs = length(graphs_with_semantic) + + IO.puts("📊 Total Graphs: #{total_graphs}") + IO.puts("🏷️ Graphs with Semantic Data: #{semantic_graphs}") + IO.puts("") + + if semantic_graphs == 0 do + IO.puts("ℹ️ No semantic annotations found. Add semantic metadata to your components:") + IO.puts(" |> text(\"Hello\", semantic: Semantic.text_buffer(buffer_id: 1))") + IO.puts(" |> rect({100, 40}, semantic: Semantic.button(\"Save\"))") + else + IO.puts("📋 All Graphs:") + render_graph_tree(all_entries) + end + + IO.puts("") + IO.puts("💡 Use inspect_graph(\"graph_id\") to see details of a specific graph") + + :ok + else + error -> + IO.puts("Error: #{inspect(error)}") + :error + end + end + + @doc """ + Inspect a specific graph in detail. + + Shows the full semantic structure of a single graph, like zooming in + on one component in the browser dev tools. + + ## Examples + + iex> inspect_graph("ABC123...") + === Graph Details: ABC123... === + + 🏷️ Semantic Elements: 5 + 📅 Last Updated: 2024-01-15 14:30:22 + + 📝 text_buffer (1): + └── Element #7: %{buffer_id: "uuid1", editable: true, role: :textbox} + Content: "Hello, World!" + """ + def inspect_graph(graph_key, viewport_name \\ :main_viewport) do + with {:ok, viewport} <- get_viewport(viewport_name) do + case :ets.lookup(viewport.semantic_table, graph_key) do + [{^graph_key, data}] -> + IO.puts("=== Graph Details: #{String.slice(graph_key, 0, 8)}... ===") + IO.puts("") + + element_count = map_size(data.elements) + timestamp = format_timestamp(data.timestamp) + + IO.puts("🏷️ Semantic Elements: #{element_count}") + IO.puts("📅 Last Updated: #{timestamp}") + IO.puts("") + + if element_count == 0 do + IO.puts("⚪ No semantic elements in this graph") + else + render_semantic_details(data) + end + + :ok + + [] -> + IO.puts("❌ Graph not found: #{graph_key}") + + # Show available graphs + all_entries = :ets.tab2list(viewport.semantic_table) + IO.puts("\n📋 Available graphs:") + Enum.each(all_entries, fn {key, _data} -> + short_key = String.slice(key, 0, 8) <> "..." + IO.puts(" #{short_key}") + end) + + :error + end + else + error -> + IO.puts("Error: #{inspect(error)}") + :error + end + end + + @doc """ + Show just the semantic content - like a simplified view. + + Perfect for beginners who just want to see "what semantic stuff is in my app?" + + ## Examples + + iex> show_semantic() + === Semantic Content in Your App === + + 📝 Text Buffers (2): + • Buffer "uuid1": "Hello, World!" + • Buffer "uuid2": "def hello do..." + + 🔘 Buttons (3): + • "Save" + • "Cancel" + • "Submit" + """ + def show_semantic(viewport_name \\ :main_viewport) do + with {:ok, viewport} <- get_viewport(viewport_name) do + IO.puts("=== Semantic Content in Your App ===") + IO.puts("") + + # Collect all semantic elements across all graphs + all_entries = :ets.tab2list(viewport.semantic_table) + + all_elements = Enum.flat_map(all_entries, fn {_key, data} -> + Map.values(data.elements) + end) + + if all_elements == [] do + IO.puts("🚫 No semantic content found") + IO.puts("") + IO.puts("💡 To add semantic annotations to your components:") + IO.puts(" |> text(\"Hello\", semantic: Semantic.text_buffer(buffer_id: 1))") + IO.puts(" |> rect({100, 40}, semantic: Semantic.button(\"Save\"))") + else + # Group by semantic type + by_type = Enum.group_by(all_elements, fn elem -> + elem.semantic.type + end) + + # Show each type + Enum.each(by_type, fn {type, elements} -> + count = length(elements) + icon = get_type_icon(type) + type_name = String.capitalize(to_string(type)) <> "s" + + IO.puts("#{icon} #{type_name} (#{count}):") + + Enum.each(elements, fn elem -> + description = format_element_description(elem) + IO.puts(" • #{description}") + end) + + IO.puts("") + end) + end + + :ok + else + error -> + IO.puts("Error: #{inspect(error)}") + :error + end + end + + # ============================================================================= + # Private Helpers + # ============================================================================= + + defp render_graph_tree(all_entries) do + {graphs_with_semantic, graphs_without} = Enum.split_with(all_entries, fn {_key, data} -> + map_size(data.elements) > 0 + end) + + # Show graphs with semantic data first + Enum.with_index(graphs_with_semantic, fn {key, data}, index -> + is_last_semantic = index == length(graphs_with_semantic) - 1 + has_more_graphs = length(graphs_without) > 0 + + connector = if is_last_semantic and not has_more_graphs, do: "└──", else: "├──" + + short_key = String.slice(key, 0, 8) <> "..." + element_count = map_size(data.elements) + + IO.puts("#{connector} 🏷️ \"#{short_key}\" (#{element_count} semantic elements)") + + # Show a preview of elements + data.elements + |> Map.values() + |> Enum.take(3) + |> Enum.with_index() + |> Enum.each(fn {elem, elem_index} -> + is_last_elem = elem_index == min(2, element_count - 1) + elem_connector = if is_last_semantic and not has_more_graphs and is_last_elem, do: " └──", else: "│ ├──" + + icon = get_type_icon(elem.semantic.type) + description = format_element_brief(elem) + IO.puts("#{elem_connector} #{icon} #{description}") + end) + + if element_count > 3 do + more_connector = if is_last_semantic and not has_more_graphs, do: " └──", else: "│ └──" + IO.puts("#{more_connector} ... and #{element_count - 3} more") + end + end) + + # Show graphs without semantic data + Enum.with_index(graphs_without, fn {key, _data}, index -> + is_last = index == length(graphs_without) - 1 + connector = if is_last, do: "└──", else: "├──" + + short_key = String.slice(key, 0, 8) <> "..." + IO.puts("#{connector} ⚪ \"#{short_key}\" (no semantic data)") + end) + end + + defp render_semantic_details(data) do + Enum.each(data.by_type, fn {type, element_ids} -> + icon = get_type_icon(type) + type_name = String.capitalize(to_string(type)) + count = length(element_ids) + + IO.puts("#{icon} #{type_name} (#{count}):") + + Enum.each(element_ids, fn id -> + elem = Map.get(data.elements, id) + description = format_element_description(elem) + IO.puts(" └── Element ##{id}: #{description}") + + # Show content if it's a text element + if elem.content && String.trim(elem.content) != "" do + content_preview = String.slice(elem.content, 0, 50) + content_preview = if String.length(elem.content) > 50, do: content_preview <> "...", else: content_preview + IO.puts(" Content: #{inspect(content_preview)}") + end + end) + + IO.puts("") + end) + end + + defp format_element_brief(elem) do + case elem.semantic.type do + :text_buffer -> "text_buffer: buffer_id \"#{String.slice(elem.semantic.buffer_id, 0, 8)}...\"" + :button -> "button: \"#{elem.semantic.label}\"" + :menu -> "menu: \"#{elem.semantic.name}\"" + :text_input -> "text_input: \"#{elem.semantic.name}\"" + other -> "#{other}: #{inspect(elem.semantic)}" + end + end + + defp format_element_description(elem) do + case elem.semantic.type do + :text_buffer -> + id = String.slice(elem.semantic.buffer_id, 0, 8) <> "..." + preview = if elem.content && String.trim(elem.content) != "" do + " - \"#{String.slice(elem.content, 0, 20)}...\"" + else + " (empty)" + end + "Buffer #{id}#{preview}" + + :button -> "\"#{elem.semantic.label}\"" + :menu -> "\"#{elem.semantic.name}\" menu" + :text_input -> "\"#{elem.semantic.name}\" input" + _other -> "#{inspect(elem.semantic)}" + end + end + + defp get_type_icon(type) do + case type do + :text_buffer -> "📝" + :button -> "🔘" + :menu -> "📂" + :text_input -> "📝" + _ -> "🏷️" + end + end + + defp format_size({width, height}), do: "#{width}x#{height}" + defp format_size(_), do: "unknown" + + defp format_timestamp(timestamp) when is_integer(timestamp) do + # Convert from milliseconds since Unix epoch + datetime = DateTime.from_unix!(timestamp, :millisecond) + Calendar.strftime(datetime, "%Y-%m-%d %H:%M:%S") + end + defp format_timestamp(_), do: "unknown" defp get_viewport(viewport) when is_struct(viewport, ViewPort), do: {:ok, viewport} defp get_viewport(name) when is_atom(name) do diff --git a/lib/scenic/dev_tools_enhanced.ex b/lib/scenic/dev_tools_enhanced.ex deleted file mode 100644 index 3b6e029c..00000000 --- a/lib/scenic/dev_tools_enhanced.ex +++ /dev/null @@ -1,143 +0,0 @@ -defmodule Scenic.DevToolsEnhanced do - @moduledoc """ - Enhanced developer tools that work with UUID graph keys. - - This module extends the basic DevTools to handle cases where - graph keys are UUIDs rather than simple atoms. - """ - - alias Scenic.ViewPort - - @doc """ - Display all semantic information across all graphs. - """ - def semantic_all(viewport_name \\ :main_viewport) do - with {:ok, viewport} <- get_viewport(viewport_name) do - entries = :ets.tab2list(viewport.semantic_table) - - # Filter to only entries with semantic data - semantic_entries = Enum.filter(entries, fn {_key, data} -> - map_size(data.elements) > 0 - end) - - if semantic_entries == [] do - IO.puts("No semantic information found in any graph") - else - IO.puts("=== Semantic Information Across All Graphs ===") - IO.puts("Found #{length(semantic_entries)} graphs with semantic data\n") - - Enum.each(semantic_entries, fn {graph_key, data} -> - IO.puts("Graph: #{graph_key}") - IO.puts("Elements: #{map_size(data.elements)}") - - if map_size(data.by_type) > 0 do - IO.puts("By type:") - Enum.each(data.by_type, fn {type, ids} -> - IO.puts(" #{type}: #{length(ids)} element(s)") - - # Show details for each element - Enum.each(ids, fn id -> - elem = Map.get(data.elements, id) - case type do - :text_buffer -> - IO.puts(" - Buffer #{elem.semantic.buffer_id}") - content_preview = String.slice(elem.content || "", 0, 50) - if content_preview != "", do: IO.puts(" Content: #{inspect(content_preview)}") - - :button -> - IO.puts(" - Button: #{elem.semantic.label}") - - _ -> - IO.puts(" - #{inspect(elem.semantic)}") - end - end) - end) - end - IO.puts("") - end) - end - :ok - end - end - - @doc """ - Show all text buffers across all graphs. - """ - def buffers_all(viewport_name \\ :main_viewport) do - with {:ok, viewport} <- get_viewport(viewport_name) do - entries = :ets.tab2list(viewport.semantic_table) - - # Find all text buffers - all_buffers = Enum.flat_map(entries, fn {graph_key, data} -> - buffer_ids = Map.get(data.by_type, :text_buffer, []) - - Enum.map(buffer_ids, fn id -> - elem = Map.get(data.elements, id) - %{ - graph_key: graph_key, - buffer_id: elem.semantic.buffer_id, - content: elem.content || "", - semantic: elem.semantic - } - end) - end) - - if all_buffers == [] do - IO.puts("No text buffers found") - else - IO.puts("Text Buffers:") - Enum.each(all_buffers, fn buffer -> - content_preview = String.slice(buffer.content, 0, 60) - content_preview = if String.length(buffer.content) > 60, do: content_preview <> "...", else: content_preview - - IO.puts("\n[Buffer: #{buffer.buffer_id}]") - IO.puts("Graph: #{buffer.graph_key}") - IO.puts("Content: #{inspect(content_preview)}") - end) - end - :ok - end - end - - @doc """ - Get content of a specific buffer by UUID. - """ - def buffer_by_uuid(buffer_uuid, viewport_name \\ :main_viewport) do - with {:ok, viewport} <- get_viewport(viewport_name) do - entries = :ets.tab2list(viewport.semantic_table) - - # Search for the buffer - result = Enum.find_value(entries, fn {_graph_key, data} -> - buffer_ids = Map.get(data.by_type, :text_buffer, []) - - Enum.find_value(buffer_ids, fn id -> - elem = Map.get(data.elements, id) - if elem.semantic.buffer_id == buffer_uuid do - elem.content || "" - end - end) - end) - - case result do - nil -> - IO.puts("Buffer #{buffer_uuid} not found") - :error - content -> - IO.puts("Buffer #{buffer_uuid}:") - IO.puts(content) - :ok - end - end - end - - # Private helpers - - defp get_viewport(viewport) when is_struct(viewport, ViewPort), do: {:ok, viewport} - defp get_viewport(name) when is_atom(name) do - case Process.whereis(name) do - nil -> {:error, "ViewPort #{inspect(name)} not found"} - pid -> ViewPort.info(pid) - end - end - defp get_viewport(pid) when is_pid(pid), do: ViewPort.info(pid) -end \ No newline at end of file diff --git a/lib/scenic/dev_tools_inspector.ex b/lib/scenic/dev_tools_inspector.ex deleted file mode 100644 index 7aad8724..00000000 --- a/lib/scenic/dev_tools_inspector.ex +++ /dev/null @@ -1,341 +0,0 @@ -defmodule Scenic.DevToolsInspector do - @moduledoc """ - High-level Scenic application inspector - like browser dev tools for Scenic. - - Provides a hierarchical view of your Scenic application starting from the root, - making it easy to understand the scene structure and semantic annotations. - """ - - alias Scenic.ViewPort - - @doc """ - Inspect the entire Scenic application hierarchy. - - Shows a tree-like view starting from the ViewPort, listing all scenes/graphs - and highlighting which ones have semantic annotations. - - ## Examples - - iex> Scenic.DevToolsInspector.inspect_app() - === Scenic Application Inspector === - ViewPort: :main_viewport (1440x855) - - 📊 Total Graphs: 8 - 🏷️ Graphs with Semantic Data: 2 - - 📋 All Graphs: - ├── 🏷️ "ABC123..." (5 semantic elements) - │ ├── 📝 text_buffer: buffer_id "uuid1" - │ ├── 🔘 button: "Save" - │ └── 📂 menu: "File" - ├── ⚪ "DEF456..." (no semantic data) - ├── 🏷️ "GHI789..." (1 semantic element) - │ └── 📝 text_buffer: buffer_id "uuid2" - └── ⚪ "JKL012..." (no semantic data) - """ - def inspect_app(viewport_name \\ :main_viewport) do - with {:ok, viewport} <- get_viewport(viewport_name) do - IO.puts("=== Scenic Application Inspector ===") - IO.puts("ViewPort: #{inspect(viewport_name)} (#{format_size(viewport.size)})") - IO.puts("") - - # Get all semantic data - all_entries = :ets.tab2list(viewport.semantic_table) - - graphs_with_semantic = Enum.filter(all_entries, fn {_key, data} -> - map_size(data.elements) > 0 - end) - - total_graphs = length(all_entries) - semantic_graphs = length(graphs_with_semantic) - - IO.puts("📊 Total Graphs: #{total_graphs}") - IO.puts("🏷️ Graphs with Semantic Data: #{semantic_graphs}") - IO.puts("") - - if semantic_graphs == 0 do - IO.puts("ℹ️ No semantic annotations found. Add semantic metadata to your components:") - IO.puts(" |> text(\"Hello\", semantic: Semantic.text_buffer(buffer_id: 1))") - IO.puts(" |> rect({100, 40}, semantic: Semantic.button(\"Save\"))") - else - IO.puts("📋 All Graphs:") - render_graph_tree(all_entries) - end - - IO.puts("") - IO.puts("💡 Use inspect_graph(\"graph_id\") to see details of a specific graph") - - :ok - else - error -> - IO.puts("Error: #{inspect(error)}") - :error - end - end - - @doc """ - Inspect a specific graph in detail. - - Shows the full semantic structure of a single graph, like zooming in - on one component in the browser dev tools. - - ## Examples - - iex> Scenic.DevToolsInspector.inspect_graph("ABC123...") - === Graph Details: ABC123... === - - 🏷️ Semantic Elements: 5 - 📅 Last Updated: 2024-01-15 14:30:22 - - 📝 text_buffer (1): - └── Element #7: %{buffer_id: "uuid1", editable: true, role: :textbox} - Content: "Hello, World!" - - 🔘 button (2): - └── Element #12: %{label: "Save"} - └── Element #15: %{label: "Cancel"} - """ - def inspect_graph(graph_key, viewport_name \\ :main_viewport) do - with {:ok, viewport} <- get_viewport(viewport_name) do - case :ets.lookup(viewport.semantic_table, graph_key) do - [{^graph_key, data}] -> - IO.puts("=== Graph Details: #{String.slice(graph_key, 0, 8)}... ===") - IO.puts("") - - element_count = map_size(data.elements) - timestamp = format_timestamp(data.timestamp) - - IO.puts("🏷️ Semantic Elements: #{element_count}") - IO.puts("📅 Last Updated: #{timestamp}") - IO.puts("") - - if element_count == 0 do - IO.puts("⚪ No semantic elements in this graph") - else - render_semantic_details(data) - end - - :ok - - [] -> - IO.puts("❌ Graph not found: #{graph_key}") - - # Show available graphs - all_entries = :ets.tab2list(viewport.semantic_table) - IO.puts("\n📋 Available graphs:") - Enum.each(all_entries, fn {key, _data} -> - short_key = String.slice(key, 0, 8) <> "..." - IO.puts(" #{short_key}") - end) - - :error - end - else - error -> - IO.puts("Error: #{inspect(error)}") - :error - end - end - - @doc """ - Show just the semantic content - like a simplified view. - - Perfect for beginners who just want to see "what semantic stuff is in my app?" - - ## Examples - - iex> Scenic.DevToolsInspector.show_semantic_content() - === Semantic Content in Your App === - - 📝 Text Buffers (2): - • Buffer "uuid1": "Hello, World!" - • Buffer "uuid2": "def hello do..." - - 🔘 Buttons (3): - • "Save" - • "Cancel" - • "Submit" - - 📂 Menus (1): - • "File" menu - """ - def show_semantic_content(viewport_name \\ :main_viewport) do - with {:ok, viewport} <- get_viewport(viewport_name) do - IO.puts("=== Semantic Content in Your App ===") - IO.puts("") - - # Collect all semantic elements across all graphs - all_entries = :ets.tab2list(viewport.semantic_table) - - all_elements = Enum.flat_map(all_entries, fn {_key, data} -> - Map.values(data.elements) - end) - - if all_elements == [] do - IO.puts("🚫 No semantic content found") - IO.puts("") - IO.puts("💡 To add semantic annotations to your components:") - IO.puts(" |> text(\"Hello\", semantic: Semantic.text_buffer(buffer_id: 1))") - IO.puts(" |> rect({100, 40}, semantic: Semantic.button(\"Save\"))") - else - # Group by semantic type - by_type = Enum.group_by(all_elements, fn elem -> - elem.semantic.type - end) - - # Show each type - Enum.each(by_type, fn {type, elements} -> - count = length(elements) - icon = get_type_icon(type) - type_name = String.capitalize(to_string(type)) <> "s" - - IO.puts("#{icon} #{type_name} (#{count}):") - - Enum.each(elements, fn elem -> - description = format_element_description(elem) - IO.puts(" • #{description}") - end) - - IO.puts("") - end) - end - - :ok - else - error -> - IO.puts("Error: #{inspect(error)}") - :error - end - end - - # Private helpers - - defp render_graph_tree(all_entries) do - {graphs_with_semantic, graphs_without} = Enum.split_with(all_entries, fn {_key, data} -> - map_size(data.elements) > 0 - end) - - # Show graphs with semantic data first - Enum.with_index(graphs_with_semantic, fn {key, data}, index -> - is_last_semantic = index == length(graphs_with_semantic) - 1 - has_more_graphs = length(graphs_without) > 0 - - connector = if is_last_semantic and not has_more_graphs, do: "└──", else: "├──" - - short_key = String.slice(key, 0, 8) <> "..." - element_count = map_size(data.elements) - - IO.puts("#{connector} 🏷️ \"#{short_key}\" (#{element_count} semantic elements)") - - # Show a preview of elements - data.elements - |> Map.values() - |> Enum.take(3) - |> Enum.with_index() - |> Enum.each(fn {elem, elem_index} -> - is_last_elem = elem_index == min(2, element_count - 1) - elem_connector = if is_last_semantic and not has_more_graphs and is_last_elem, do: " └──", else: "│ ├──" - - icon = get_type_icon(elem.semantic.type) - description = format_element_brief(elem) - IO.puts("#{elem_connector} #{icon} #{description}") - end) - - if element_count > 3 do - more_connector = if is_last_semantic and not has_more_graphs, do: " └──", else: "│ └──" - IO.puts("#{more_connector} ... and #{element_count - 3} more") - end - end) - - # Show graphs without semantic data - Enum.with_index(graphs_without, fn {key, _data}, index -> - is_last = index == length(graphs_without) - 1 - connector = if is_last, do: "└──", else: "├──" - - short_key = String.slice(key, 0, 8) <> "..." - IO.puts("#{connector} ⚪ \"#{short_key}\" (no semantic data)") - end) - end - - defp render_semantic_details(data) do - Enum.each(data.by_type, fn {type, element_ids} -> - icon = get_type_icon(type) - type_name = String.capitalize(to_string(type)) - count = length(element_ids) - - IO.puts("#{icon} #{type_name} (#{count}):") - - Enum.each(element_ids, fn id -> - elem = Map.get(data.elements, id) - description = format_element_description(elem) - IO.puts(" └── Element ##{id}: #{description}") - - # Show content if it's a text element - if elem.content && String.trim(elem.content) != "" do - content_preview = String.slice(elem.content, 0, 50) - content_preview = if String.length(elem.content) > 50, do: content_preview <> "...", else: content_preview - IO.puts(" Content: #{inspect(content_preview)}") - end - end) - - IO.puts("") - end) - end - - defp format_element_brief(elem) do - case elem.semantic.type do - :text_buffer -> "text_buffer: buffer_id \"#{String.slice(elem.semantic.buffer_id, 0, 8)}...\"" - :button -> "button: \"#{elem.semantic.label}\"" - :menu -> "menu: \"#{elem.semantic.name}\"" - :text_input -> "text_input: \"#{elem.semantic.name}\"" - other -> "#{other}: #{inspect(elem.semantic)}" - end - end - - defp format_element_description(elem) do - case elem.semantic.type do - :text_buffer -> - id = String.slice(elem.semantic.buffer_id, 0, 8) <> "..." - preview = if elem.content && String.trim(elem.content) != "" do - " - \"#{String.slice(elem.content, 0, 20)}...\"" - else - " (empty)" - end - "Buffer #{id}#{preview}" - - :button -> "\"#{elem.semantic.label}\"" - :menu -> "\"#{elem.semantic.name}\" menu" - :text_input -> "\"#{elem.semantic.name}\" input" - _other -> "#{inspect(elem.semantic)}" - end - end - - defp get_type_icon(type) do - case type do - :text_buffer -> "📝" - :button -> "🔘" - :menu -> "📂" - :text_input -> "📝" - _ -> "🏷️" - end - end - - defp format_size({width, height}), do: "#{width}x#{height}" - defp format_size(_), do: "unknown" - - defp format_timestamp(timestamp) when is_integer(timestamp) do - # Convert from milliseconds since Unix epoch - datetime = DateTime.from_unix!(timestamp, :millisecond) - Calendar.strftime(datetime, "%Y-%m-%d %H:%M:%S") - end - defp format_timestamp(_), do: "unknown" - - defp get_viewport(viewport) when is_struct(viewport, ViewPort), do: {:ok, viewport} - defp get_viewport(name) when is_atom(name) do - case Process.whereis(name) do - nil -> {:error, "ViewPort #{inspect(name)} not found"} - pid -> ViewPort.info(pid) - end - end - defp get_viewport(pid) when is_pid(pid), do: ViewPort.info(pid) -end \ No newline at end of file From 8fd4909c945c2981813dae7acf978b7d988ad568 Mon Sep 17 00:00:00 2001 From: JediLuke <1879634+JediLuke@users.noreply.github.com> Date: Sun, 13 Jul 2025 23:31:59 -0500 Subject: [PATCH 05/11] wip --- SEMANTIC_DOM.md | 144 ++++++++++ lib/scenic/dev_tools.ex | 583 ++++++++++++++++++++++------------------ 2 files changed, 470 insertions(+), 257 deletions(-) create mode 100644 SEMANTIC_DOM.md diff --git a/SEMANTIC_DOM.md b/SEMANTIC_DOM.md new file mode 100644 index 00000000..1b34c993 --- /dev/null +++ b/SEMANTIC_DOM.md @@ -0,0 +1,144 @@ +# Scenic Semantic DOM + +This document describes the semantic DOM system added to Scenic, which enables testing and development tools to query GUI elements by their semantic meaning rather than visual properties. + +## Overview + +The semantic DOM provides a parallel data structure alongside Scenic's rendering pipeline that stores metadata about GUI elements. This allows tools to: + +- Query elements by type (button, text_buffer, menu, etc.) +- Access element properties without parsing visual output +- Build testing frameworks that understand GUI semantics +- Create developer tools for inspecting running applications + +## Architecture + +### Core Components + +1. **ViewPort Enhancement** (`lib/scenic/view_port.ex`) + - Added `semantic_table` ETS table to store semantic data per graph + - Added `get_semantic/2` and `inspect_semantic/2` functions + - Semantic data is stored alongside script data + +2. **Semantic Helper Module** (`lib/scenic/semantic.ex`) + - Provides convenience functions for creating semantic annotations + - Common patterns: `text_buffer/1`, `button/1`, `menu/1`, etc. + +3. **Query API** (`lib/scenic/semantic/query.ex`) + - Functions to query semantic data: `find_by_type/3`, `get_buffer_text/3`, etc. + - Returns semantic elements with their properties and content + +4. **Developer Tools** (`lib/scenic/dev_tools.ex`) + - Generic tools for any Scenic application + - Scene hierarchy visualization + - Graph inspection + - Semantic element queries + +## Usage + +### Adding Semantic Annotations + +When building graphs, add semantic metadata to primitives: + +```elixir +@graph Graph.build() +|> text("Hello World", + id: :my_text, + semantic: Semantic.text_buffer(buffer_id: "buffer_1")) +|> rect({100, 40}, + id: :save_btn, + semantic: Semantic.button("Save")) +``` + +### Querying Semantic Data + +In tests or development tools: + +```elixir +# Find all buttons +{:ok, buttons} = Semantic.Query.find_by_type(viewport, :button) + +# Get text buffer content +{:ok, content} = Semantic.Query.get_buffer_text(viewport, "buffer_1") + +# Use developer tools +import Scenic.DevTools +scene_tree() # Show scene hierarchy +semantic_summary() # Show all semantic elements +find_semantic(:button) # Find elements by type +``` + +### Application-Specific Tools + +Applications can build their own semantic tools on top of the generic ones: + +```elixir +defmodule MyApp.DevTools do + # Use Scenic's generic tools + alias Scenic.DevTools + + # Add app-specific functionality + def show_active_document() do + # Custom logic using semantic queries + end +end +``` + +## Semantic Types + +Common semantic types include: + +- `:text_buffer` - Text editing areas +- `:button` - Clickable buttons +- `:menu` - Menu items or menu bars +- `:text_input` - Single-line text inputs +- `:list` - List containers +- `:list_item` - Individual list items + +Applications can define custom semantic types as needed. + +## Best Practices + +1. **Always add semantic data to interactive elements** - Buttons, inputs, menus should have semantic annotations + +2. **Use consistent IDs** - Buffer IDs, element IDs should be stable across renders + +3. **Include relevant metadata** - Add properties that tests might need (editable, role, file_path, etc.) + +4. **Keep semantic data lightweight** - Don't duplicate large content, reference it + +5. **Update semantic data with content** - When text changes, update the semantic content + +## Integration with Testing + +The semantic DOM enables powerful testing patterns: + +```elixir +# In a test +test "clicking save button saves the document" do + # Start your scene + {:ok, scene} = MyScene.start_link() + + # Query semantic elements + {:ok, save_button} = find_one(viewport, :button, fn b -> + b.semantic.label == "Save" + end) + + # Simulate interaction + send_event(scene, {:click, save_button.id}) + + # Verify results via semantic queries + {:ok, content} = get_buffer_text(viewport, "main_buffer") + assert content == "Expected content" +end +``` + +## Future Enhancements + +Potential future improvements: + +- Automatic semantic annotation inference +- Semantic change notifications +- Integration with accessibility tools +- Cross-graph semantic relationships +- Semantic-based event routing \ No newline at end of file diff --git a/lib/scenic/dev_tools.ex b/lib/scenic/dev_tools.ex index 4a9b8a84..5f84dfb9 100644 --- a/lib/scenic/dev_tools.ex +++ b/lib/scenic/dev_tools.ex @@ -1,134 +1,64 @@ defmodule Scenic.DevTools do @moduledoc """ - Comprehensive developer tools for inspecting Scenic applications during development. + Generic developer tools for inspecting Scenic applications during development. - This module combines basic semantic inspection, UUID-aware enhanced tools, - and high-level "browser dev tools" style inspection into a single namespace. + This module provides tools for understanding the structure and state of any + Scenic application, focusing on the graph hierarchy and semantic annotations. ## Usage in IEx iex> import Scenic.DevTools # High-level inspection (recommended) - iex> inspect_app() # Browser dev tools style view - iex> show_semantic() # Simple semantic content overview + iex> inspect_app() # Hierarchical view of scenes and graphs + iex> show_semantic() # All semantic content in the app - # Detailed queries - iex> semantic() # Show semantic info for default viewport - iex> buffers() # Show all text buffers - iex> buttons() # Show all buttons + # Scene hierarchy + iex> scene_tree() # Show scene parent-child relationships + iex> inspect_scene("_main_") # Detailed view of a specific scene - # Enhanced tools for UUID graphs - iex> semantic_all() # All semantic data across graphs - iex> buffers_all() # All buffers across graphs - """ - - alias Scenic.ViewPort - - @doc """ - Display semantic information for the current viewport. - - ## Examples - - iex> semantic() - === Semantic Tree for :main === - Total elements: 3 + # Graph inspection + iex> inspect_graph("uuid") # Detailed view of a specific graph + iex> list_graphs() # List all graphs with their scenes - By type: - text_buffer: 1 element - - :buffer_1: %{type: :text_buffer, buffer_id: 1} - button: 2 elements - - :save_btn: %{type: :button, label: "Save"} - - :cancel_btn: %{type: :button, label: "Cancel"} + # Semantic queries (generic) + iex> semantic_summary() # Summary of all semantic annotations + iex> find_semantic(:button) # Find all elements of a semantic type """ - def semantic(viewport_name \\ :main_viewport, graph_key \\ :main) do - with {:ok, viewport} <- get_viewport(viewport_name) do - ViewPort.inspect_semantic(viewport, graph_key) - else - error -> - IO.puts("Error: #{inspect(error)}") - :error - end - end - - @doc """ - Show all text buffers and their content. - ## Examples - - iex> buffers() - Text Buffers: - [1] "Hello, World!" - [2] "def my_function do\\n :ok\\nend" - """ - def buffers(viewport_name \\ :main_viewport, graph_key \\ :main) do - with {:ok, viewport} <- get_viewport(viewport_name), - {:ok, buffers} <- Scenic.Semantic.Query.find_by_type(viewport, :text_buffer, graph_key) do - IO.puts("Text Buffers:") - Enum.each(buffers, fn buffer -> - content = buffer.content || "" - buffer_id = buffer.semantic.buffer_id - preview = String.slice(content, 0, 60) - preview = if String.length(content) > 60, do: preview <> "...", else: preview - IO.puts("[#{buffer_id}] #{inspect(preview)}") - end) - :ok - else - {:error, :not_found} -> - IO.puts("No text buffers found") - :ok - error -> - IO.puts("Error: #{inspect(error)}") - :error - end - end + alias Scenic.ViewPort @doc """ - Show content of a specific buffer. + Show the scene hierarchy as a tree structure. - ## Examples - - iex> buffer(1) - Buffer 1: - Hello, World! - """ - def buffer(buffer_id, viewport_name \\ :main_viewport, graph_key \\ :main) do - with {:ok, viewport} <- get_viewport(viewport_name), - {:ok, content} <- Scenic.Semantic.Query.get_buffer_text(viewport, buffer_id, graph_key) do - IO.puts("Buffer #{buffer_id}:") - IO.puts(content) - :ok - else - error -> - IO.puts("Error: #{inspect(error)}") - :error - end - end - - @doc """ - Show all buttons in the viewport. + Displays the parent-child relationships between scenes, starting from + the root scene and showing all descendant scenes. ## Examples - iex> buttons() - Buttons: - - "Save" (id: :save_btn) - - "Cancel" (id: :cancel_btn) + iex> scene_tree() + === Scene Hierarchy === + 📊 Root Scene: "_main_" (Flamelex.GUI.RootScene) + ├── 📄 "layer_1" (Flamelex.GUI.Layers.Layer1) + ├── 📄 "layer_2" (Flamelex.GUI.Layers.Layer2) + │ └── 📄 "buffer_pane_abc123" (Quillex.BufferPane) + └── 📄 "layer_4" (Flamelex.GUI.Layers.Layer4) """ - def buttons(viewport_name \\ :main_viewport, graph_key \\ :main) do - with {:ok, viewport} <- get_viewport(viewport_name), - {:ok, buttons} <- Scenic.Semantic.Query.get_buttons(viewport, graph_key) do - IO.puts("Buttons:") - Enum.each(buttons, fn button -> - label = button.semantic.label - id = button.id - IO.puts("- #{inspect(label)} (id: #{inspect(id)})") - end) + def scene_tree(viewport_name \\ :main_viewport) do + with {:ok, viewport} <- get_viewport(viewport_name) do + IO.puts("=== Scene Hierarchy ===") + + # Build the scene tree + tree = build_scene_tree(viewport) + + if tree do + render_scene_node(tree, "") + else + IO.puts("No scenes found in viewport") + end + :ok else - {:error, :not_found} -> - IO.puts("No buttons found") - :ok error -> IO.puts("Error: #{inspect(error)}") :error @@ -136,26 +66,67 @@ defmodule Scenic.DevTools do end @doc """ - Find elements by semantic type. + List all graphs showing which scene owns each graph. ## Examples - iex> find(:menu) - Found 1 menu element(s): - - :main_menu: %{type: :menu, name: "File"} + iex> list_graphs() + === Graphs in ViewPort === + Total graphs: 5 + + Graph "_root_" (root scene graph) + Scene: "_main_" (Flamelex.GUI.RootScene) + Has semantic data: Yes (3 elements) + + Graph "abc123..." + Scene: "buffer_pane_1" (Quillex.BufferPane) + Has semantic data: Yes (1 element) """ - def find(type, viewport_name \\ :main_viewport, graph_key \\ :main) do - with {:ok, viewport} <- get_viewport(viewport_name), - {:ok, elements} <- Scenic.Semantic.Query.find_by_type(viewport, type, graph_key) do - IO.puts("Found #{length(elements)} #{type} element(s):") - Enum.each(elements, fn elem -> - IO.puts("- #{inspect(elem.id)}: #{inspect(elem.semantic)}") + def list_graphs(viewport_name \\ :main_viewport) do + with {:ok, viewport} <- get_viewport(viewport_name) do + IO.puts("=== Graphs in ViewPort ===") + + # Get all scripts from the script table + scripts = :ets.tab2list(viewport.script_table) + semantic_entries = :ets.tab2list(viewport.semantic_table) + + IO.puts("Total graphs: #{length(scripts)}") + IO.puts("") + + # Group scripts by their scene + Enum.each(scripts, fn {graph_id, _script, owner_pid} -> + # Find scene info for this owner + scene_info = find_scene_by_pid(viewport, owner_pid) + + # Check if has semantic data + semantic_info = Enum.find(semantic_entries, fn {id, _data} -> id == graph_id end) + has_semantic = case semantic_info do + {_, data} -> map_size(data.elements) > 0 + nil -> false + end + + graph_label = if graph_id == "_root_", do: "(root scene graph)", else: "" + IO.puts("Graph \"#{short_id(graph_id)}\" #{graph_label}") + + case scene_info do + {scene_id, module} -> + IO.puts(" Scene: \"#{scene_id}\" (#{inspect(module)})") + nil -> + IO.puts(" Scene: Unknown (pid: #{inspect(owner_pid)})") + end + + if has_semantic do + {_, data} = semantic_info + IO.puts(" Has semantic data: Yes (#{map_size(data.elements)} elements)") + else + IO.puts(" Has semantic data: No") + end + + IO.puts("") end) + :ok else - {:error, :not_found} -> - IO.puts("No #{type} elements found") - :ok error -> IO.puts("Error: #{inspect(error)}") :error @@ -163,67 +134,23 @@ defmodule Scenic.DevTools do end @doc """ - Get raw semantic data for a viewport. - - ## Examples + Show a summary of all semantic annotations in the application. - iex> raw_semantic() - %{ - elements: %{...}, - by_type: %{...}, - timestamp: 1234567890 - } - """ - def raw_semantic(viewport_name \\ :main_viewport, graph_key \\ :main) do - with {:ok, viewport} <- get_viewport(viewport_name), - {:ok, info} <- ViewPort.get_semantic(viewport, graph_key) do - info - else - error -> error - end - end - - @doc """ - List all semantic types in use. + Groups semantic elements by type across all graphs. ## Examples - iex> types() - Semantic types in use: - - button (2 elements) - - text_buffer (1 element) - - menu (1 element) - """ - def types(viewport_name \\ :main_viewport, graph_key \\ :main) do - with {:ok, viewport} <- get_viewport(viewport_name), - {:ok, info} <- ViewPort.get_semantic(viewport, graph_key) do - IO.puts("Semantic types in use:") - info.by_type - |> Enum.sort_by(fn {_type, ids} -> -length(ids) end) - |> Enum.each(fn {type, ids} -> - count = length(ids) - element_word = if count == 1, do: "element", else: "elements" - IO.puts("- #{type} (#{count} #{element_word})") - end) - :ok - else - error -> - IO.puts("Error: #{inspect(error)}") - :error - end - end - - # ============================================================================= - # Enhanced Tools (UUID-aware) - # ============================================================================= - - @doc """ - Display all semantic information across all graphs. - - This function works with UUID graph keys and shows semantic data - from all graphs in the viewport. + iex> semantic_summary() + === Semantic Summary === + Total semantic elements: 15 across 3 graphs + + By type: + button: 5 elements + text_input: 3 elements + menu: 2 elements + custom_widget: 5 elements """ - def semantic_all(viewport_name \\ :main_viewport) do + def semantic_summary(viewport_name \\ :main_viewport) do with {:ok, viewport} <- get_viewport(viewport_name) do entries = :ets.tab2list(viewport.semantic_table) @@ -232,120 +159,175 @@ defmodule Scenic.DevTools do map_size(data.elements) > 0 end) - if semantic_entries == [] do - IO.puts("No semantic information found in any graph") + # Collect all semantic elements across all graphs + all_elements = Enum.flat_map(semantic_entries, fn {_graph_key, data} -> + Map.values(data.elements) + end) + + total_elements = length(all_elements) + graph_count = length(semantic_entries) + + IO.puts("=== Semantic Summary ===") + IO.puts("Total semantic elements: #{total_elements} across #{graph_count} graphs") + IO.puts("") + + if total_elements == 0 do + IO.puts("No semantic annotations found.") + IO.puts("Add semantic metadata to your components:") + IO.puts(" |> rect({100, 40}, semantic: %{type: :button, label: \"Save\"})") else - IO.puts("=== Semantic Information Across All Graphs ===") - IO.puts("Found #{length(semantic_entries)} graphs with semantic data\n") + # Group by type + by_type = Enum.group_by(all_elements, fn elem -> elem.semantic.type end) - Enum.each(semantic_entries, fn {graph_key, data} -> - IO.puts("Graph: #{graph_key}") - IO.puts("Elements: #{map_size(data.elements)}") - - if map_size(data.by_type) > 0 do - IO.puts("By type:") - Enum.each(data.by_type, fn {type, ids} -> - IO.puts(" #{type}: #{length(ids)} element(s)") - - # Show details for each element - Enum.each(ids, fn id -> - elem = Map.get(data.elements, id) - case type do - :text_buffer -> - IO.puts(" - Buffer #{elem.semantic.buffer_id}") - content_preview = String.slice(elem.content || "", 0, 50) - if content_preview != "", do: IO.puts(" Content: #{inspect(content_preview)}") - - :button -> - IO.puts(" - Button: #{elem.semantic.label}") - - _ -> - IO.puts(" - #{inspect(elem.semantic)}") - end - end) - end) - end - IO.puts("") + IO.puts("By type:") + by_type + |> Enum.sort_by(fn {_type, elems} -> -length(elems) end) + |> Enum.each(fn {type, elements} -> + IO.puts(" #{type}: #{length(elements)} elements") end) end + :ok end end @doc """ - Show all text buffers across all graphs. + Find all semantic elements of a given type across all graphs. + + ## Examples + + iex> find_semantic(:button) + === Elements of type :button === + Found 3 elements: + + Graph "abc123...": + - %{type: :button, label: "Save"} + - %{type: :button, label: "Cancel"} + + Graph "def456...": + - %{type: :button, label: "Submit"} """ - def buffers_all(viewport_name \\ :main_viewport) do + def find_semantic(type, viewport_name \\ :main_viewport) do with {:ok, viewport} <- get_viewport(viewport_name) do entries = :ets.tab2list(viewport.semantic_table) - # Find all text buffers - all_buffers = Enum.flat_map(entries, fn {graph_key, data} -> - buffer_ids = Map.get(data.by_type, :text_buffer, []) + # Find all elements of this type + results = Enum.flat_map(entries, fn {graph_key, data} -> + element_ids = Map.get(data.by_type, type, []) - Enum.map(buffer_ids, fn id -> + elements = Enum.map(element_ids, fn id -> elem = Map.get(data.elements, id) - %{ - graph_key: graph_key, - buffer_id: elem.semantic.buffer_id, - content: elem.content || "", - semantic: elem.semantic - } + {graph_key, elem} end) + + if elements == [], do: [], else: [{graph_key, elements}] end) - if all_buffers == [] do - IO.puts("No text buffers found") + IO.puts("=== Elements of type #{inspect(type)} ===") + + if results == [] do + IO.puts("No elements found") else - IO.puts("Text Buffers:") - Enum.each(all_buffers, fn buffer -> - content_preview = String.slice(buffer.content, 0, 60) - content_preview = if String.length(buffer.content) > 60, do: content_preview <> "...", else: content_preview - - IO.puts("\n[Buffer: #{buffer.buffer_id}]") - IO.puts("Graph: #{buffer.graph_key}") - IO.puts("Content: #{inspect(content_preview)}") + total = Enum.reduce(results, 0, fn {_, elems}, acc -> acc + length(elems) end) + IO.puts("Found #{total} elements:") + IO.puts("") + + Enum.each(results, fn {graph_key, elements} -> + IO.puts("Graph \"#{short_id(graph_key)}\":") + Enum.each(elements, fn {_graph_key, elem} -> + IO.puts(" - #{inspect(elem.semantic)}") + end) + IO.puts("") end) end + :ok + else + error -> + IO.puts("Error: #{inspect(error)}") + :error end end @doc """ - Get content of a specific buffer by UUID. + Inspect a specific scene showing its graph and semantic data. + + ## Examples + + iex> inspect_scene("_main_") + === Scene: "_main_" === + Module: Flamelex.GUI.RootScene + Graph ID: "_root_" + Child scenes: 4 + + Semantic elements in graph: + - button: "File" menu + - button: "Edit" menu """ - def buffer_by_uuid(buffer_uuid, viewport_name \\ :main_viewport) do + def inspect_scene(scene_id, viewport_name \\ :main_viewport) do with {:ok, viewport} <- get_viewport(viewport_name) do - entries = :ets.tab2list(viewport.semantic_table) - - # Search for the buffer - result = Enum.find_value(entries, fn {_graph_key, data} -> - buffer_ids = Map.get(data.by_type, :text_buffer, []) - - Enum.find_value(buffer_ids, fn id -> - elem = Map.get(data.elements, id) - if elem.semantic.buffer_id == buffer_uuid do - elem.content || "" - end - end) - end) - - case result do - nil -> - IO.puts("Buffer #{buffer_uuid} not found") + case Map.get(viewport.scenes_by_id, scene_id) do + nil -> + IO.puts("Scene \"#{scene_id}\" not found") + available = Map.keys(viewport.scenes_by_id) + IO.puts("\nAvailable scenes: #{inspect(available)}") :error - content -> - IO.puts("Buffer #{buffer_uuid}:") - IO.puts(content) + + {pid, parent_pid} -> + # Get scene info from pid map + {^scene_id, _parent_id, module} = Map.get(viewport.scenes_by_pid, pid) + + IO.puts("=== Scene: \"#{scene_id}\" ===") + IO.puts("Module: #{inspect(module)}") + IO.puts("PID: #{inspect(pid)}") + if parent_pid, do: IO.puts("Parent PID: #{inspect(parent_pid)}") + + # Find the graph for this scene + scripts = :ets.tab2list(viewport.script_table) + scene_graphs = Enum.filter(scripts, fn {_id, _script, owner} -> owner == pid end) + + if scene_graphs != [] do + IO.puts("\nGraphs owned by this scene:") + Enum.each(scene_graphs, fn {graph_id, _script, _owner} -> + IO.puts(" Graph ID: \"#{short_id(graph_id)}\"") + + # Check semantic data + case :ets.lookup(viewport.semantic_table, graph_id) do + [{^graph_id, data}] when map_size(data.elements) > 0 -> + IO.puts(" Semantic elements:") + Enum.each(data.by_type, fn {type, ids} -> + IO.puts(" - #{type}: #{length(ids)} element(s)") + end) + _ -> + IO.puts(" No semantic data") + end + end) + end + + # Find child scenes + children = Enum.filter(viewport.scenes_by_pid, fn {child_pid, _} -> + case Map.get(viewport.scenes_by_id, Map.get(viewport.scenes_by_pid, child_pid) |> elem(0)) do + {^child_pid, ^pid} -> true + _ -> false + end + end) + + if children != [] do + IO.puts("\nChild scenes: #{length(children)}") + Enum.each(children, fn {_child_pid, {child_id, _, child_module}} -> + IO.puts(" - \"#{child_id}\" (#{inspect(child_module)})") + end) + end + :ok end + else + error -> + IO.puts("Error: #{inspect(error)}") + :error end end - # ============================================================================= - # High-Level Inspector (Browser Dev Tools Style) - # ============================================================================= - @doc """ Inspect your entire Scenic application like browser dev tools. @@ -540,6 +522,79 @@ defmodule Scenic.DevTools do # Private Helpers # ============================================================================= + # Private helper to build scene tree + defp build_scene_tree(viewport) do + # Find root scene + root_entry = Enum.find(viewport.scenes_by_id, fn {id, _} -> + id == "_main_" + end) + + case root_entry do + {root_id, {root_pid, _parent}} -> + {^root_id, _parent_id, module} = Map.get(viewport.scenes_by_pid, root_pid) + build_scene_node(root_id, root_pid, module, viewport) + nil -> + nil + end + end + + defp build_scene_node(id, pid, module, viewport) do + # Find children of this scene + children = Enum.filter(viewport.scenes_by_id, fn {_child_id, {_child_pid, parent_pid}} -> + parent_pid == pid + end) + + # Build child nodes + child_nodes = Enum.map(children, fn {child_id, {child_pid, _}} -> + {^child_id, _, child_module} = Map.get(viewport.scenes_by_pid, child_pid) + build_scene_node(child_id, child_pid, child_module, viewport) + end) + + %{ + id: id, + pid: pid, + module: module, + children: child_nodes + } + end + + defp render_scene_node(node, prefix) do + # Determine if this is the last child at this level + is_root = prefix == "" + + # Render this node + icon = if node.id == "_main_", do: "📊", else: "📄" + label = if node.id == "_main_", do: "Root Scene: ", else: "" + + if is_root do + IO.puts("#{icon} #{label}\"#{node.id}\" (#{inspect(node.module)})") + else + IO.puts("#{prefix}#{icon} \"#{node.id}\" (#{inspect(node.module)})") + end + + # Render children + Enum.with_index(node.children, fn child, index -> + is_last = index == length(node.children) - 1 + + {child_prefix, next_prefix} = if is_root do + child_prefix = if is_last, do: "└── ", else: "├── " + next_prefix = if is_last, do: " ", else: "│ " + {child_prefix, next_prefix} + else + child_prefix = if is_last, do: "#{prefix}└── ", else: "#{prefix}├── " + next_prefix = if is_last, do: "#{prefix} ", else: "#{prefix}│ " + {child_prefix, next_prefix} + end + + render_scene_node(child, child_prefix) + + # Continue with grandchildren using the appropriate prefix + Enum.each(child.children, fn grandchild -> + render_scene_node(grandchild, next_prefix) + end) + end) + end + defp render_graph_tree(all_entries) do {graphs_with_semantic, graphs_without} = Enum.split_with(all_entries, fn {_key, data} -> map_size(data.elements) > 0 @@ -660,6 +715,20 @@ defmodule Scenic.DevTools do end defp format_timestamp(_), do: "unknown" + # Helper to find scene info by pid + defp find_scene_by_pid(viewport, pid) do + case Map.get(viewport.scenes_by_pid, pid) do + {id, _parent_id, module} -> {id, module} + nil -> nil + end + end + + # Helper to shorten long IDs + defp short_id(id) when byte_size(id) > 12 do + String.slice(id, 0, 8) <> "..." + end + defp short_id(id), do: id + defp get_viewport(viewport) when is_struct(viewport, ViewPort), do: {:ok, viewport} defp get_viewport(name) when is_atom(name) do case Process.whereis(name) do From b713e747dfeac45afb913a2beecdc60255838c94 Mon Sep 17 00:00:00 2001 From: JediLuke <1879634+JediLuke@users.noreply.github.com> Date: Mon, 14 Jul 2025 01:41:08 -0500 Subject: [PATCH 06/11] wip --- DEVTOOLS_GUIDE.md | 301 ++++++ examples/semantic_component_example.ex | 247 +++++ lib/scenic/dev_tools.ex | 1284 +++++++++++++----------- 3 files changed, 1266 insertions(+), 566 deletions(-) create mode 100644 DEVTOOLS_GUIDE.md create mode 100644 examples/semantic_component_example.ex diff --git a/DEVTOOLS_GUIDE.md b/DEVTOOLS_GUIDE.md new file mode 100644 index 00000000..f7e2ae8b --- /dev/null +++ b/DEVTOOLS_GUIDE.md @@ -0,0 +1,301 @@ +# Scenic DevTools Guide + +## Overview + +The new Scenic DevTools provide a unified interface for inspecting and debugging Scenic applications, with a focus on semantic content that supports development, testing, automation, and accessibility. + +## Key Improvements + +### 1. Unified `inspect_viewport()` Function + +The new `inspect_viewport()` replaces both `scene_tree()` and `inspect_app()` with a single, powerful function that provides: + +```elixir +# Import the tools +import Scenic.DevTools + +# Full inspection (default) +inspect_viewport() + +# Options for focused inspection +inspect_viewport(show: :semantic) # Just semantic content +inspect_viewport(show: :scenes) # Just scene hierarchy +inspect_viewport(show: :graphs) # Just graphs +inspect_viewport(verbose: true) # Detailed view +inspect_viewport(graph: "abc123") # Specific graph +``` + +### 2. Semantic-First Design + +The tools are built around the semantic layer, making it easy to: +- Find elements by their semantic type +- Access content directly without parsing visual output +- Build automation and accessibility features + +### 3. Generic Semantic Functions + +```elixir +# Find elements by type +find(:button) +find(:text_buffer) +find(:menu) + +# See what types are in use +types() + +# View all semantic content +semantic() + +# Get raw data for advanced queries +raw_semantic() +``` + +### 4. Application-Specific Functions + +Text editor applications (like Quillex) extend the DevTools with specialized functions: + +```elixir +# Import both modules +import Scenic.DevTools +import Quillex.DevTools + +# Text buffer functions (Quillex-specific) +buffers() # List all text buffers with preview +buffer(1) # Show full content of buffer by index +buffer("uuid") # Show buffer by ID +buffer("file.ex") # Show buffer by filename + +# Other editor-specific tools +cursor_info() # Show cursor position and context +selection_info() # Show current selection +buffer_stats() # Detailed buffer statistics +syntax_info() # Language/syntax detection +``` + +## Semantic Annotation Best Practices + +### Adding Semantic Data + +When building Scenic graphs, add semantic annotations to make elements queryable: + +```elixir +@graph Graph.build() + # Text buffer with semantic info + |> text(buffer_content, + id: :buffer_text, + semantic: Semantic.text_buffer(buffer_id: buffer.uuid)) + + # Button with semantic info + |> rect({100, 40}, + id: :save_button, + semantic: Semantic.button("Save")) + + # Custom semantic data + |> group( + semantic: %{ + type: :code_editor, + language: :elixir, + file_path: "lib/myfile.ex", + editable: true + }) +``` + +### Semantic Helper Functions + +The `Scenic.Semantic` module provides helpers for common patterns: + +```elixir +Semantic.button("Label") # Clickable button +Semantic.text_buffer(buffer_id: id) # Text editor buffer +Semantic.text_input(name, opts) # Input field +Semantic.menu(name, opts) # Menu container +Semantic.menu_item(label, opts) # Menu item +Semantic.annotate(type, attrs) # Generic annotation +``` + +### Custom Semantic Types + +You can create custom semantic types for your application: + +```elixir +# In your component +|> rect({100, 100}, + semantic: %{ + type: :code_cell, # Custom type + language: :elixir, + cell_id: 42, + execution_state: :ready + }) + +# Query it later +find(:code_cell) +``` + +## Usage Examples + +### During Development + +```elixir +iex> import Scenic.DevTools + +# Get overview of your app +iex> inspect_viewport() + +╔═══ Scenic ViewPort Inspector ═══╗ +║ ViewPort: :main_viewport +║ Size: 800×600 +║ Root: Elixir.MyApp.Scene.Root +╚═════════════════════════════════╝ + +🎬 Scene Hierarchy: +───────────────── +🎬 _main_ (RootScene) +├── 📄 editor_1 (EditorScene) +└── 📄 sidebar (SidebarScene) + +📊 Graphs & Semantic Content: +──────────────────────────── + +🏷️ With Semantic Data: + 📋 "abc123..." (scene: editor_1) + Elements: 3 + 📝 text_buffer: 1 buffer + 🔘 button: Save, Cancel + +📈 Semantic Summary: +────────────────── + 📝 text_buffer: 1 (1 buffer) + 🔘 button: 2 (Save, Cancel) + +💡 Quick Commands: + • inspect_viewport(show: :semantic) # Just semantic content + • find(:button) # Find by type + • types() # List all types + • semantic() # All semantic content +``` + +### Finding Elements + +```elixir +iex> find(:button) + +🔘 Found 3 button element(s): + +Graph "abc123...": + • "Save" + • "Cancel" + +Graph "def456...": + • "Submit" +``` + +### Application-Specific Buffer Access (Quillex) + +```elixir +# Import Quillex-specific tools +iex> import Quillex.DevTools + +# See all buffers +iex> buffers() + +📝 Text Buffers: +[1] Buffer abc12345... (main.ex): "defmodule MyApp do..." +[2] Buffer def67890... (notes.md): "# TODO: implement feature" + +# Inspect specific buffer +iex> buffer(1) + +📝 Buffer Details: +ID: abc12345-6789-0123-4567-890123456789 +File: lib/main.ex +Editable: true + +Stats: 25 lines, 89 words, 487 chars +Cursor: Line 5, Column 12 + +--- Content --- +defmodule MyApp do + use Application + + def start(_type, _args) do + # ... + end +end +--- End --- +``` + +## Application-Specific Extensions + +Applications can extend the DevTools with domain-specific functionality: + +```elixir +defmodule MyApp.DevTools do + # Import base DevTools + import Scenic.DevTools + + # Add app-specific tools + def show_active_document() do + # Use semantic queries to find active document + case find(:document) do + # Custom logic here + end + end + + def list_open_files() do + # Query semantic data for file information + raw_semantic() + |> extract_file_info() + |> display_files() + end +end +``` + +## Integration with Testing + +The DevTools are designed to work seamlessly with testing frameworks: + +```elixir +defmodule MyAppTest do + use ExUnit.Case + alias Scenic.Semantic.Query + + test "save button saves the document" do + # Start your scene + {:ok, scene, viewport} = start_scene() + + # Use semantic queries + {:ok, save_button} = Query.get_button_by_label(viewport, "Save") + {:ok, buffer_text} = Query.get_buffer_text(viewport, buffer_id) + + # Simulate interaction + send_event(scene, {:click, save_button.id}) + + # Verify results + assert file_saved?() + end +end +``` + +## Tips and Tricks + +1. **Use verbose mode for debugging**: `inspect_viewport(verbose: true)` shows additional details + +2. **Filter by graph**: `inspect_viewport(graph: "partial_id")` to focus on specific components + +3. **Combine with IEx helpers**: The DevTools work great with IEx's autocomplete and history + +4. **Add semantic data incrementally**: Start with key interactive elements, add more as needed + +5. **Use consistent semantic types**: Stick to common types like `:button`, `:text_buffer`, `:menu` when possible + +6. **Include metadata**: Add extra fields like `file_path`, `cursor_position`, `language` to semantic data + +## Future Enhancements + +The semantic layer and DevTools are designed to support future features: + +- **Accessibility**: Screen readers can use semantic data +- **Automation**: UI testing frameworks can find elements reliably +- **AI Integration**: Language models can understand UI structure +- **Remote Debugging**: Inspect apps running on other devices +- **Time-Travel Debugging**: Record and replay semantic state changes \ No newline at end of file diff --git a/examples/semantic_component_example.ex b/examples/semantic_component_example.ex new file mode 100644 index 00000000..150497ad --- /dev/null +++ b/examples/semantic_component_example.ex @@ -0,0 +1,247 @@ +defmodule SemanticComponentExample do + @moduledoc """ + Example showing best practices for adding semantic annotations to Scenic components. + + This example demonstrates: + - How to add semantic data to primitives + - Using the Semantic helper module + - Creating custom semantic types + - Making components testable and accessible + """ + + use Scenic.Component + import Scenic.Primitives + import Scenic.Components + alias Scenic.Graph + alias Scenic.Semantic + + @impl Scenic.Component + def validate(data), do: {:ok, data} + + @impl Scenic.Component + def init(data, opts) do + # Build a graph with comprehensive semantic annotations + graph = build_graph(data) + + state = %{ + graph: graph, + data: data, + buffer_content: "Hello, Scenic!", + selected_item: nil + } + + {:ok, state, push: graph} + end + + defp build_graph(_data) do + Graph.build() + # Header with semantic menu annotation + |> group(fn g -> + g + |> rect({800, 40}, fill: :light_gray, semantic: Semantic.menu("main_menu", orientation: :horizontal)) + |> text("File", translate: {10, 25}, semantic: Semantic.menu_item("File", parent_menu: "main_menu")) + |> text("Edit", translate: {60, 25}, semantic: Semantic.menu_item("Edit", parent_menu: "main_menu")) + |> text("View", translate: {110, 25}, semantic: Semantic.menu_item("View", parent_menu: "main_menu")) + end) + + # Main editor area with text buffer semantic + |> group(fn g -> + g + |> rect({800, 400}, translate: {0, 40}, fill: :white, stroke: {1, :gray}) + |> text("Hello, Scenic!", + translate: {10, 60}, + id: :main_buffer, + semantic: Semantic.text_buffer( + buffer_id: "main_editor_buffer", + editable: true + )) + end, translate: {0, 0}) + + # Toolbar with semantic buttons + |> group(fn g -> + g + |> button("Save", + id: :save_btn, + translate: {10, 450}, + semantic: Semantic.button("Save")) + |> button("Cancel", + id: :cancel_btn, + translate: {80, 450}, + semantic: Semantic.button("Cancel")) + |> button("Run", + id: :run_btn, + translate: {170, 450}, + semantic: %{ + type: :button, + label: "Run", + action: :execute_code, + shortcut: "Cmd+R" + }) + end) + + # Sidebar with custom semantic type + |> group(fn g -> + g + |> rect({200, 400}, translate: {600, 40}, fill: :light_blue) + |> text("Files", translate: {610, 60}, font_size: 16) + |> group(fn g -> + g + |> text("• main.ex", translate: {610, 90}, + id: :file_1, + semantic: %{ + type: :file_item, + path: "lib/main.ex", + file_type: :elixir, + selectable: true + }) + |> text("• test.ex", translate: {610, 110}, + id: :file_2, + semantic: %{ + type: :file_item, + path: "test/test.ex", + file_type: :elixir, + selectable: true + }) + end) + end, semantic: %{ + type: :file_browser, + role: :navigation, + label: "Project Files" + }) + + # Status bar with semantic info + |> group(fn g -> + g + |> rect({800, 30}, translate: {0, 570}, fill: :dark_gray) + |> text("Ready", + translate: {10, 590}, + fill: :white, + semantic: %{ + type: :status_text, + live: true, + updates: :frequently + }) + |> text("Line 1, Col 1", + translate: {700, 590}, + fill: :white, + semantic: %{ + type: :cursor_position, + line: 1, + column: 1 + }) + end, semantic: Semantic.annotate(:status_bar)) + + # Search input with semantic annotation + |> text_input("", + id: :search_input, + translate: {300, 10}, + width: 200, + hint: "Search...", + semantic: Semantic.text_input("search", + placeholder: "Search files and symbols", + shortcut: "Cmd+F" + )) + end + + # Event handlers that could update semantic data + @impl Scenic.Component + def handle_event({:click, :save_btn}, _context, state) do + # Update status with semantic info + new_graph = state.graph + |> Graph.modify(:status_text, fn p -> + # Update both visual and semantic + p + |> text("Saving...") + |> update_opts(semantic: %{ + type: :status_text, + state: :saving, + live: true + }) + end) + + {:noreply, %{state | graph: new_graph}, push: new_graph} + end + + def handle_event({:click, {:file_item, path}}, _context, state) do + # Update selection state semantically + new_graph = state.graph + |> Graph.modify(:file_browser, fn p -> + p |> update_opts(semantic: %{ + type: :file_browser, + selected_file: path, + role: :navigation + }) + end) + + {:noreply, %{state | graph: new_graph, selected_item: path}, push: new_graph} + end + + def handle_event(_event, _context, state) do + {:noreply, state} + end + + @impl Scenic.Component + def handle_info({:buffer_changed, new_content}, state) do + # Update buffer content and semantic data + new_graph = state.graph + |> Graph.modify(:main_buffer, fn p -> + p + |> text(new_content) + |> update_opts(semantic: Semantic.text_buffer( + buffer_id: "main_editor_buffer", + editable: true, + modified: true, + content_hash: :erlang.phash2(new_content) + )) + end) + + {:noreply, %{state | graph: new_graph, buffer_content: new_content}, push: new_graph} + end + + def handle_info(_msg, state) do + {:noreply, state} + end +end + +defmodule SemanticComponentExample.Test do + @moduledoc """ + Example test showing how to use semantic queries with the component. + """ + + def demo_semantic_queries() do + # This would be in a real test + viewport = :main_viewport + + # Import the dev tools + import Scenic.DevTools + + IO.puts("\n=== Semantic Component Example ===\n") + + # Find all buttons + IO.puts("Finding all buttons:") + find(:button) + + # Find custom semantic types + IO.puts("\nFinding file items:") + find(:file_item) + + # Inspect the whole viewport + IO.puts("\nFull inspection:") + inspect_viewport() + + # Get raw semantic data for advanced queries + IO.puts("\nRaw semantic data for custom queries:") + data = raw_semantic() + + # Custom query example + file_items = for {_graph_key, graph_data} <- data, + {_id, element} <- graph_data.elements, + element.semantic.type == :file_item, + do: element.semantic + + IO.puts("Found #{length(file_items)} file items") + Enum.each(file_items, fn item -> + IO.puts(" - #{item.path} (#{item.file_type})") + end) + end +end \ No newline at end of file diff --git a/lib/scenic/dev_tools.ex b/lib/scenic/dev_tools.ex index 5f84dfb9..105412c0 100644 --- a/lib/scenic/dev_tools.ex +++ b/lib/scenic/dev_tools.ex @@ -1,697 +1,811 @@ defmodule Scenic.DevTools do @moduledoc """ - Generic developer tools for inspecting Scenic applications during development. + Developer tools for inspecting Scenic applications during development. - This module provides tools for understanding the structure and state of any - Scenic application, focusing on the graph hierarchy and semantic annotations. + This module provides a unified interface for understanding the structure, + state, and semantic content of Scenic applications. It's designed for + development, testing, automation, and accessibility. - ## Usage in IEx + ## Quick Start iex> import Scenic.DevTools - - # High-level inspection (recommended) - iex> inspect_app() # Hierarchical view of scenes and graphs - iex> show_semantic() # All semantic content in the app - - # Scene hierarchy - iex> scene_tree() # Show scene parent-child relationships - iex> inspect_scene("_main_") # Detailed view of a specific scene - - # Graph inspection - iex> inspect_graph("uuid") # Detailed view of a specific graph - iex> list_graphs() # List all graphs with their scenes - - # Semantic queries (generic) - iex> semantic_summary() # Summary of all semantic annotations - iex> find_semantic(:button) # Find all elements of a semantic type + iex> inspect_viewport() # The main inspection function + + ## Other Tools + + iex> semantic() # Just semantic content + iex> buffers() # Text buffer shortcuts + iex> find(:button) # Find elements by type """ alias Scenic.ViewPort @doc """ - Show the scene hierarchy as a tree structure. + The primary inspection function for Scenic applications. - Displays the parent-child relationships between scenes, starting from - the root scene and showing all descendant scenes. + Provides a unified view of: + - ViewPort configuration + - Scene hierarchy with relationships + - Graphs with their semantic content + - Interactive elements summary + + This replaces both scene_tree() and inspect_app() with a cleaner, + more informative display. + + ## Options + + * `:viewport` - ViewPort name or pid (default: `:main_viewport`) + * `:graph` - Specific graph to inspect (default: all graphs) + * `:show` - What to display: `:all`, `:scenes`, `:semantic`, `:graphs` + * `:verbose` - Show additional details (default: false) ## Examples - iex> scene_tree() - === Scene Hierarchy === - 📊 Root Scene: "_main_" (Flamelex.GUI.RootScene) - ├── 📄 "layer_1" (Flamelex.GUI.Layers.Layer1) - ├── 📄 "layer_2" (Flamelex.GUI.Layers.Layer2) - │ └── 📄 "buffer_pane_abc123" (Quillex.BufferPane) - └── 📄 "layer_4" (Flamelex.GUI.Layers.Layer4) - """ - def scene_tree(viewport_name \\ :main_viewport) do - with {:ok, viewport} <- get_viewport(viewport_name) do - IO.puts("=== Scene Hierarchy ===") + # Default view - everything + iex> inspect_viewport() - # Build the scene tree - tree = build_scene_tree(viewport) + # Just semantic content + iex> inspect_viewport(show: :semantic) - if tree do - render_scene_node(tree, "") - else - IO.puts("No scenes found in viewport") + # Specific graph with details + iex> inspect_viewport(graph: "abc123", verbose: true) + + # Different viewport + iex> inspect_viewport(viewport: :secondary) + """ + def inspect_viewport(opts \\ []) do + viewport_name = Keyword.get(opts, :viewport, :main_viewport) + show = Keyword.get(opts, :show, :all) + graph_filter = Keyword.get(opts, :graph) + verbose = Keyword.get(opts, :verbose, false) + + with {:ok, viewport} <- get_viewport(viewport_name) do + IO.puts("\n╔═══ Scenic ViewPort Inspector ═══╗") + IO.puts("║ ViewPort: #{inspect(viewport_name)}") + IO.puts("║ Size: #{format_size(viewport.size)}") + IO.puts("╚═════════════════════════════════╝\n") + + case show do + :all -> + show_hierarchy_if_available(viewport) + show_graphs_with_semantic(viewport, graph_filter, verbose) + show_semantic_summary(viewport) + + :hierarchy -> + show_hierarchy_if_available(viewport) + + :scenes -> + show_hierarchy_if_available(viewport) + + :semantic -> + show_graphs_with_semantic(viewport, graph_filter, verbose) + show_semantic_summary(viewport) + + :graphs -> + show_graphs_with_semantic(viewport, graph_filter, verbose) end + show_quick_tips() :ok else error -> - IO.puts("Error: #{inspect(error)}") + IO.puts("❌ Error: #{inspect(error)}") :error end end @doc """ - List all graphs showing which scene owns each graph. - - ## Examples + Show all semantic content across the application. - iex> list_graphs() - === Graphs in ViewPort === - Total graphs: 5 - - Graph "_root_" (root scene graph) - Scene: "_main_" (Flamelex.GUI.RootScene) - Has semantic data: Yes (3 elements) - - Graph "abc123..." - Scene: "buffer_pane_1" (Quillex.BufferPane) - Has semantic data: Yes (1 element) + A simplified view focusing just on the semantic elements, + organized by type with content previews. """ - def list_graphs(viewport_name \\ :main_viewport) do + def semantic(viewport_name \\ :main_viewport, graph_key \\ nil) do with {:ok, viewport} <- get_viewport(viewport_name) do - IO.puts("=== Graphs in ViewPort ===") - - # Get all scripts from the script table - scripts = :ets.tab2list(viewport.script_table) - semantic_entries = :ets.tab2list(viewport.semantic_table) - - IO.puts("Total graphs: #{length(scripts)}") - IO.puts("") - - # Group scripts by their scene - Enum.each(scripts, fn {graph_id, _script, owner_pid} -> - # Find scene info for this owner - scene_info = find_scene_by_pid(viewport, owner_pid) - - # Check if has semantic data - semantic_info = Enum.find(semantic_entries, fn {id, _data} -> id == graph_id end) - has_semantic = case semantic_info do - {_, data} -> map_size(data.elements) > 0 - nil -> false - end - - graph_label = if graph_id == "_root_", do: "(root scene graph)", else: "" - IO.puts("Graph \"#{short_id(graph_id)}\" #{graph_label}") - - case scene_info do - {scene_id, module} -> - IO.puts(" Scene: \"#{scene_id}\" (#{inspect(module)})") - nil -> - IO.puts(" Scene: Unknown (pid: #{inspect(owner_pid)})") - end - - if has_semantic do - {_, data} = semantic_info - IO.puts(" Has semantic data: Yes (#{map_size(data.elements)} elements)") - else - IO.puts(" Has semantic data: No") - end - - IO.puts("") - end) - + if graph_key do + # Show semantic for specific graph + show_graph_semantic(viewport, graph_key) + else + # Show all semantic content + show_all_semantic(viewport) + end :ok - else - error -> - IO.puts("Error: #{inspect(error)}") - :error end end - @doc """ - Show a summary of all semantic annotations in the application. - Groups semantic elements by type across all graphs. + @doc """ + Find elements by semantic type. ## Examples - iex> semantic_summary() - === Semantic Summary === - Total semantic elements: 15 across 3 graphs - - By type: - button: 5 elements - text_input: 3 elements - menu: 2 elements - custom_widget: 5 elements + iex> find(:button) + iex> find(:text_buffer) + iex> find(:menu) """ - def semantic_summary(viewport_name \\ :main_viewport) do + def find(type, viewport_name \\ :main_viewport) do with {:ok, viewport} <- get_viewport(viewport_name) do entries = :ets.tab2list(viewport.semantic_table) - # Filter to only entries with semantic data - semantic_entries = Enum.filter(entries, fn {_key, data} -> - map_size(data.elements) > 0 - end) - - # Collect all semantic elements across all graphs - all_elements = Enum.flat_map(semantic_entries, fn {_graph_key, data} -> - Map.values(data.elements) - end) - - total_elements = length(all_elements) - graph_count = length(semantic_entries) - - IO.puts("=== Semantic Summary ===") - IO.puts("Total semantic elements: #{total_elements} across #{graph_count} graphs") - IO.puts("") + elements = for {graph_key, data} <- entries, + id <- Map.get(data.by_type, type, []), + elem = Map.get(data.elements, id), + do: {graph_key, elem} - if total_elements == 0 do - IO.puts("No semantic annotations found.") - IO.puts("Add semantic metadata to your components:") - IO.puts(" |> rect({100, 40}, semantic: %{type: :button, label: \"Save\"})") + if elements == [] do + IO.puts("No #{type} elements found") + show_available_types(entries) else - # Group by type - by_type = Enum.group_by(all_elements, fn elem -> elem.semantic.type end) + icon = get_type_icon(type) + IO.puts("\n#{icon} Found #{length(elements)} #{type} element(s):") - IO.puts("By type:") - by_type - |> Enum.sort_by(fn {_type, elems} -> -length(elems) end) - |> Enum.each(fn {type, elements} -> - IO.puts(" #{type}: #{length(elements)} elements") + Enum.group_by(elements, fn {graph_key, _} -> graph_key end) + |> Enum.each(fn {graph_key, elems} -> + IO.puts("\nGraph \"#{short_id(graph_key)}\":") + Enum.each(elems, fn {_, elem} -> + desc = format_element_for_find(elem) + IO.puts(" • #{desc}") + end) end) end - :ok end end @doc """ - Find all semantic elements of a given type across all graphs. - - ## Examples - - iex> find_semantic(:button) - === Elements of type :button === - Found 3 elements: - - Graph "abc123...": - - %{type: :button, label: "Save"} - - %{type: :button, label: "Cancel"} - - Graph "def456...": - - %{type: :button, label: "Submit"} + List all semantic types in use. """ - def find_semantic(type, viewport_name \\ :main_viewport) do + def types(viewport_name \\ :main_viewport) do with {:ok, viewport} <- get_viewport(viewport_name) do entries = :ets.tab2list(viewport.semantic_table) - # Find all elements of this type - results = Enum.flat_map(entries, fn {graph_key, data} -> - element_ids = Map.get(data.by_type, type, []) - - elements = Enum.map(element_ids, fn id -> - elem = Map.get(data.elements, id) - {graph_key, elem} + type_counts = entries + |> Enum.flat_map(fn {_, data} -> + Enum.map(data.by_type, fn {type, ids} -> {type, length(ids)} end) + end) + |> Enum.reduce(%{}, fn {type, count}, acc -> + Map.update(acc, type, count, &(&1 + count)) end) - - if elements == [], do: [], else: [{graph_key, elements}] - end) - - IO.puts("=== Elements of type #{inspect(type)} ===") - if results == [] do - IO.puts("No elements found") + if map_size(type_counts) == 0 do + IO.puts("No semantic types found") else - total = Enum.reduce(results, 0, fn {_, elems}, acc -> acc + length(elems) end) - IO.puts("Found #{total} elements:") - IO.puts("") - - Enum.each(results, fn {graph_key, elements} -> - IO.puts("Graph \"#{short_id(graph_key)}\":") - Enum.each(elements, fn {_graph_key, elem} -> - IO.puts(" - #{inspect(elem.semantic)}") - end) - IO.puts("") + IO.puts("\n🏷️ Semantic Types in Use:") + type_counts + |> Enum.sort_by(fn {_, count} -> -count end) + |> Enum.each(fn {type, count} -> + icon = get_type_icon(type) + IO.puts("#{icon} #{type}: #{count} element(s)") end) end - :ok - else - error -> - IO.puts("Error: #{inspect(error)}") - :error end end @doc """ - Inspect a specific scene showing its graph and semantic data. + Get raw semantic data for advanced queries. - ## Examples - - iex> inspect_scene("_main_") - === Scene: "_main_" === - Module: Flamelex.GUI.RootScene - Graph ID: "_root_" - Child scenes: 4 - - Semantic elements in graph: - - button: "File" menu - - button: "Edit" menu + Returns the semantic data structure that you can + query programmatically. """ - def inspect_scene(scene_id, viewport_name \\ :main_viewport) do + def raw_semantic(viewport_name \\ :main_viewport, graph_key \\ nil) do with {:ok, viewport} <- get_viewport(viewport_name) do - case Map.get(viewport.scenes_by_id, scene_id) do - nil -> - IO.puts("Scene \"#{scene_id}\" not found") - available = Map.keys(viewport.scenes_by_id) - IO.puts("\nAvailable scenes: #{inspect(available)}") - :error - - {pid, parent_pid} -> - # Get scene info from pid map - {^scene_id, _parent_id, module} = Map.get(viewport.scenes_by_pid, pid) - - IO.puts("=== Scene: \"#{scene_id}\" ===") - IO.puts("Module: #{inspect(module)}") - IO.puts("PID: #{inspect(pid)}") - if parent_pid, do: IO.puts("Parent PID: #{inspect(parent_pid)}") - - # Find the graph for this scene - scripts = :ets.tab2list(viewport.script_table) - scene_graphs = Enum.filter(scripts, fn {_id, _script, owner} -> owner == pid end) - - if scene_graphs != [] do - IO.puts("\nGraphs owned by this scene:") - Enum.each(scene_graphs, fn {graph_id, _script, _owner} -> - IO.puts(" Graph ID: \"#{short_id(graph_id)}\"") - - # Check semantic data - case :ets.lookup(viewport.semantic_table, graph_id) do - [{^graph_id, data}] when map_size(data.elements) > 0 -> - IO.puts(" Semantic elements:") - Enum.each(data.by_type, fn {type, ids} -> - IO.puts(" - #{type}: #{length(ids)} element(s)") - end) - _ -> - IO.puts(" No semantic data") - end - end) - end - - # Find child scenes - children = Enum.filter(viewport.scenes_by_pid, fn {child_pid, _} -> - case Map.get(viewport.scenes_by_id, Map.get(viewport.scenes_by_pid, child_pid) |> elem(0)) do - {^child_pid, ^pid} -> true - _ -> false - end - end) - - if children != [] do - IO.puts("\nChild scenes: #{length(children)}") - Enum.each(children, fn {_child_pid, {child_id, _, child_module}} -> - IO.puts(" - \"#{child_id}\" (#{inspect(child_module)})") - end) - end - - :ok + if graph_key do + case :ets.lookup(viewport.semantic_table, graph_key) do + [{^graph_key, data}] -> data + [] -> nil + end + else + # Return all semantic data + entries = :ets.tab2list(viewport.semantic_table) + Map.new(entries) end - else - error -> - IO.puts("Error: #{inspect(error)}") - :error end end - + @doc """ - Inspect your entire Scenic application like browser dev tools. + Get enhanced scene script data with hierarchy and metadata. - Shows a hierarchical view starting from the ViewPort with all scenes, - graphs, and semantic annotations. Perfect for understanding your app structure. + This provides access to the enhanced scene_script layer that includes + all elements (not just semantic ones), hierarchy relationships, + and automation-friendly metadata. ## Examples - iex> inspect_app() - === Scenic Application Inspector === - ViewPort: :main_viewport (1440x855) - - 📊 Total Graphs: 8 - 🏷️ Graphs with Semantic Data: 2 + # Get all scene scripts + iex> raw_scene_script() - 📋 All Graphs: - ├── 🏷️ "ABC123..." (5 semantic elements) - │ ├── 📝 text_buffer: buffer_id "uuid1" - │ ├── 🔘 button: "Save" - │ └── 📂 menu: "File" - ├── ⚪ "DEF456..." (no semantic data) + # Get specific graph's script data + iex> raw_scene_script(:main_viewport, "graph_key") """ - def inspect_app(viewport_name \\ :main_viewport) do + def raw_scene_script(viewport_name \\ :main_viewport, graph_key \\ nil) do with {:ok, viewport} <- get_viewport(viewport_name) do - IO.puts("=== Scenic Application Inspector ===") - IO.puts("ViewPort: #{inspect(viewport_name)} (#{format_size(viewport.size)})") - IO.puts("") - - # Get all semantic data - all_entries = :ets.tab2list(viewport.semantic_table) - - graphs_with_semantic = Enum.filter(all_entries, fn {_key, data} -> - map_size(data.elements) > 0 - end) - - total_graphs = length(all_entries) - semantic_graphs = length(graphs_with_semantic) - - IO.puts("📊 Total Graphs: #{total_graphs}") - IO.puts("🏷️ Graphs with Semantic Data: #{semantic_graphs}") - IO.puts("") - - if semantic_graphs == 0 do - IO.puts("ℹ️ No semantic annotations found. Add semantic metadata to your components:") - IO.puts(" |> text(\"Hello\", semantic: Semantic.text_buffer(buffer_id: 1))") - IO.puts(" |> rect({100, 40}, semantic: Semantic.button(\"Save\"))") + if graph_key do + case :ets.lookup(viewport.scene_script_table, graph_key) do + [{^graph_key, data}] -> data + [] -> nil + end else - IO.puts("📋 All Graphs:") - render_graph_tree(all_entries) + # Return all scene script data + entries = :ets.tab2list(viewport.scene_script_table) + Map.new(entries) end - - IO.puts("") - IO.puts("💡 Use inspect_graph(\"graph_id\") to see details of a specific graph") - - :ok - else - error -> - IO.puts("Error: #{inspect(error)}") - :error end end - + @doc """ - Inspect a specific graph in detail. - - Shows the full semantic structure of a single graph, like zooming in - on one component in the browser dev tools. + Show the hierarchical structure of graphs. - ## Examples - - iex> inspect_graph("ABC123...") - === Graph Details: ABC123... === - - 🏷️ Semantic Elements: 5 - 📅 Last Updated: 2024-01-15 14:30:22 - - 📝 text_buffer (1): - └── Element #7: %{buffer_id: "uuid1", editable: true, role: :textbox} - Content: "Hello, World!" + This displays the parent-child relationships between graphs, + providing a tree view of how your application is structured. """ - def inspect_graph(graph_key, viewport_name \\ :main_viewport) do + def hierarchy(viewport_name \\ :main_viewport) do with {:ok, viewport} <- get_viewport(viewport_name) do - case :ets.lookup(viewport.semantic_table, graph_key) do - [{^graph_key, data}] -> - IO.puts("=== Graph Details: #{String.slice(graph_key, 0, 8)}... ===") - IO.puts("") - - element_count = map_size(data.elements) - timestamp = format_timestamp(data.timestamp) - - IO.puts("🏷️ Semantic Elements: #{element_count}") - IO.puts("📅 Last Updated: #{timestamp}") - IO.puts("") - - if element_count == 0 do - IO.puts("⚪ No semantic elements in this graph") - else - render_semantic_details(data) - end - - :ok - - [] -> - IO.puts("❌ Graph not found: #{graph_key}") - - # Show available graphs - all_entries = :ets.tab2list(viewport.semantic_table) - IO.puts("\n📋 Available graphs:") - Enum.each(all_entries, fn {key, _data} -> - short_key = String.slice(key, 0, 8) <> "..." - IO.puts(" #{short_key}") + entries = :ets.tab2list(viewport.scene_script_table) + + if entries == [] do + IO.puts("📊 No graphs found") + else + IO.puts("\n🏗️ Graph Hierarchy:") + IO.puts("──────────────────") + + # Find root graphs (no parent) + roots = Enum.filter(entries, fn {_, data} -> + data.parent == nil + end) + + if roots == [] do + IO.puts(" No root graphs found") + else + Enum.each(roots, fn {key, data} -> + render_hierarchy_tree(key, data, entries, "", false) end) - - :error + end end - else - error -> - IO.puts("Error: #{inspect(error)}") - :error + :ok end end - + @doc """ - Show just the semantic content - like a simplified view. + Find elements across all graphs with advanced filtering. - Perfect for beginners who just want to see "what semantic stuff is in my app?" + Supports finding by: + - Semantic type (`:text_buffer`, `:button`) + - Accessibility role (`:textbox`, `:button`) + - Primitive type (`Scenic.Primitive.Text`) + - Graph location ## Examples - iex> show_semantic() - === Semantic Content in Your App === + # Find by semantic type + iex> find_element(type: :text_buffer) - 📝 Text Buffers (2): - • Buffer "uuid1": "Hello, World!" - • Buffer "uuid2": "def hello do..." + # Find by role + iex> find_element(role: :button) - 🔘 Buttons (3): - • "Save" - • "Cancel" - • "Submit" + # Find by primitive type + iex> find_element(primitive: Scenic.Primitive.Text) + + # Find within specific graph + iex> find_element(type: :button, in_graph: "main_graph") """ - def show_semantic(viewport_name \\ :main_viewport) do + def find_element(opts, viewport_name \\ :main_viewport) do + type = Keyword.get(opts, :type) + role = Keyword.get(opts, :role) + primitive = Keyword.get(opts, :primitive) + in_graph = Keyword.get(opts, :in_graph) + with {:ok, viewport} <- get_viewport(viewport_name) do - IO.puts("=== Semantic Content in Your App ===") - IO.puts("") - - # Collect all semantic elements across all graphs - all_entries = :ets.tab2list(viewport.semantic_table) + entries = :ets.tab2list(viewport.scene_script_table) - all_elements = Enum.flat_map(all_entries, fn {_key, data} -> - Map.values(data.elements) - end) - - if all_elements == [] do - IO.puts("🚫 No semantic content found") - IO.puts("") - IO.puts("💡 To add semantic annotations to your components:") - IO.puts(" |> text(\"Hello\", semantic: Semantic.text_buffer(buffer_id: 1))") - IO.puts(" |> rect({100, 40}, semantic: Semantic.button(\"Save\"))") - else - # Group by semantic type - by_type = Enum.group_by(all_elements, fn elem -> - elem.semantic.type - end) - - # Show each type - Enum.each(by_type, fn {type, elements} -> - count = length(elements) - icon = get_type_icon(type) - type_name = String.capitalize(to_string(type)) <> "s" - - IO.puts("#{icon} #{type_name} (#{count}):") - - Enum.each(elements, fn elem -> - description = format_element_description(elem) - IO.puts(" • #{description}") - end) - - IO.puts("") + # Filter by graph if specified + entries = if in_graph do + Enum.filter(entries, fn {key, _} -> + String.contains?(to_string(key), in_graph) end) + else + entries end + # Find matching elements + elements = find_matching_elements(entries, type: type, role: role, primitive: primitive) + + display_found_elements(elements, opts) :ok - else - error -> - IO.puts("Error: #{inspect(error)}") - :error end end - # ============================================================================= - # Private Helpers - # ============================================================================= - - # Private helper to build scene tree - defp build_scene_tree(viewport) do - # Find root scene - root_entry = Enum.find(viewport.scenes_by_id, fn {id, _} -> - id == "_main_" - end) + # ============================================================================ + # Private Display Functions + # ============================================================================ + + defp show_hierarchy_if_available(viewport) do + # Check if scene_script_table is available and has data + entries = :ets.tab2list(viewport.scene_script_table) - case root_entry do - {root_id, {root_pid, _parent}} -> - {^root_id, _parent_id, module} = Map.get(viewport.scenes_by_pid, root_pid) - build_scene_node(root_id, root_pid, module, viewport) - nil -> - nil + if entries != [] do + IO.puts("🏗️ Graph Hierarchy:") + IO.puts("──────────────────") + + # Find root graphs (no parent) + roots = Enum.filter(entries, fn {_, data} -> + data.parent == nil + end) + + if roots == [] do + IO.puts(" No root graphs found") + else + Enum.each(roots, fn {key, data} -> + render_hierarchy_tree(key, data, entries, "", false) + end) + end + IO.puts("") + else + IO.puts("⚠️ Graph hierarchy data not available") + IO.puts(" (Scene script enhancement not yet populated)") + IO.puts("") end end - defp build_scene_node(id, pid, module, viewport) do - # Find children of this scene - children = Enum.filter(viewport.scenes_by_id, fn {_child_id, {_child_pid, parent_pid}} -> - parent_pid == pid - end) - - # Build child nodes - child_nodes = Enum.map(children, fn {child_id, {child_pid, _}} -> - {^child_id, _, child_module} = Map.get(viewport.scenes_by_pid, child_pid) - build_scene_node(child_id, child_pid, child_module, viewport) - end) - - %{ - id: id, - pid: pid, - module: module, - children: child_nodes - } - end - defp render_scene_node(node, prefix) do - # Determine if this is the last child at this level - is_root = prefix == "" - - # Render this node - icon = if node.id == "_main_", do: "📊", else: "📄" - label = if node.id == "_main_", do: "Root Scene: ", else: "" + defp show_graphs_with_semantic(viewport, graph_filter, verbose) do + entries = :ets.tab2list(viewport.semantic_table) - if is_root do - IO.puts("#{icon} #{label}\"#{node.id}\" (#{inspect(node.module)})") + # Filter if specific graph requested + entries = if graph_filter do + Enum.filter(entries, fn {key, _} -> + String.contains?(key, graph_filter) + end) else - IO.puts("#{prefix}#{icon} \"#{node.id}\" (#{inspect(node.module)})") + entries end - # Render children - Enum.with_index(node.children, fn child, index -> - is_last = index == length(node.children) - 1 - - {child_prefix, next_prefix} = if is_root do - child_prefix = if is_last, do: "└── ", else: "├── " - next_prefix = if is_last, do: " ", else: "│ " - {child_prefix, next_prefix} + if entries == [] do + if graph_filter do + IO.puts("📊 No graphs matching: #{graph_filter}") else - child_prefix = if is_last, do: "#{prefix}└── ", else: "#{prefix}├── " - next_prefix = if is_last, do: "#{prefix} ", else: "#{prefix}│ " - {child_prefix, next_prefix} + IO.puts("📊 No graphs found") end + else + IO.puts("📊 Graphs & Semantic Content:") + IO.puts("────────────────────────────") - render_scene_node(child, child_prefix) - - # Continue with grandchildren using the appropriate prefix - Enum.each(child.children, fn grandchild -> - render_scene_node(grandchild, next_prefix) + # Group by whether they have semantic content + {with_semantic, without} = Enum.split_with(entries, fn {_, data} -> + map_size(data.elements) > 0 end) - end) + + # Show graphs with semantic content first + if with_semantic != [] do + IO.puts("\n🏷️ With Semantic Data:") + Enum.each(with_semantic, fn {key, data} -> + render_graph_semantic(key, data, verbose, viewport) + end) + end + + # Then graphs without + if without != [] and verbose do + IO.puts("\n⚪ Without Semantic Data:") + Enum.each(without, fn {key, _} -> + IO.puts(" • Graph \"#{short_id(key)}\"") + end) + end + + IO.puts("") + end end - defp render_graph_tree(all_entries) do - {graphs_with_semantic, graphs_without} = Enum.split_with(all_entries, fn {_key, data} -> - map_size(data.elements) > 0 - end) + defp show_semantic_summary(viewport) do + entries = :ets.tab2list(viewport.semantic_table) + + all_elements = for {_, data} <- entries, + {_, elem} <- data.elements, + do: elem - # Show graphs with semantic data first - Enum.with_index(graphs_with_semantic, fn {key, data}, index -> - is_last_semantic = index == length(graphs_with_semantic) - 1 - has_more_graphs = length(graphs_without) > 0 - - connector = if is_last_semantic and not has_more_graphs, do: "└──", else: "├──" - - short_key = String.slice(key, 0, 8) <> "..." - element_count = map_size(data.elements) - - IO.puts("#{connector} 🏷️ \"#{short_key}\" (#{element_count} semantic elements)") - - # Show a preview of elements - data.elements - |> Map.values() - |> Enum.take(3) - |> Enum.with_index() - |> Enum.each(fn {elem, elem_index} -> - is_last_elem = elem_index == min(2, element_count - 1) - elem_connector = if is_last_semantic and not has_more_graphs and is_last_elem, do: " └──", else: "│ ├──" + if all_elements != [] do + IO.puts("📈 Semantic Summary:") + IO.puts("──────────────────") + + # Group by type and show counts + all_elements + |> Enum.group_by(& &1.semantic.type) + |> Enum.sort_by(fn {_, elems} -> -length(elems) end) + |> Enum.each(fn {type, elems} -> + icon = get_type_icon(type) + count = length(elems) - icon = get_type_icon(elem.semantic.type) - description = format_element_brief(elem) - IO.puts("#{elem_connector} #{icon} #{description}") + # Show sample for common types + sample = case type do + :button -> + labels = elems + |> Enum.map(& &1.semantic[:label]) + |> Enum.filter(& &1) + |> Enum.take(3) + if labels != [], do: " (#{Enum.join(labels, ", ")})", else: "" + + :text_buffer -> + " (#{count} buffer#{if count > 1, do: "s", else: ""})" + + _ -> "" + end + + IO.puts(" #{icon} #{type}: #{count}#{sample}") end) - if element_count > 3 do - more_connector = if is_last_semantic and not has_more_graphs, do: " └──", else: "│ └──" - IO.puts("#{more_connector} ... and #{element_count - 3} more") - end - end) + IO.puts("") + end + end + + defp show_quick_tips do + IO.puts("💡 Quick Commands:") + IO.puts(" • inspect_viewport(show: :semantic) # Just semantic content") + IO.puts(" • hierarchy() # Show graph structure") + IO.puts(" • find(:button) # Find by semantic type") + IO.puts(" • find_element(role: :textbox) # Find by accessibility role") + IO.puts(" • types() # List all semantic types") + IO.puts(" • semantic() # All semantic content") + IO.puts(" • raw_scene_script() # Enhanced data with hierarchy") + end + + # ============================================================================ + # Private Semantic Display Functions + # ============================================================================ + + defp show_graph_semantic(viewport, graph_key) do + case :ets.lookup(viewport.semantic_table, graph_key) do + [{^graph_key, data}] -> + IO.puts("\n📊 Semantic Data for Graph \"#{short_id(graph_key)}\":") + if map_size(data.elements) == 0 do + IO.puts(" No semantic elements") + else + render_semantic_elements(data) + end + [] -> + IO.puts("Graph not found: #{graph_key}") + end + end + + defp show_all_semantic(viewport) do + entries = :ets.tab2list(viewport.semantic_table) - # Show graphs without semantic data - Enum.with_index(graphs_without, fn {key, _data}, index -> - is_last = index == length(graphs_without) - 1 - connector = if is_last, do: "└──", else: "├──" - - short_key = String.slice(key, 0, 8) <> "..." - IO.puts("#{connector} ⚪ \"#{short_key}\" (no semantic data)") - end) + all_elements = for {_, data} <- entries, + {_, elem} <- data.elements, + do: elem + + if all_elements == [] do + IO.puts("\n🚫 No semantic content found") + IO.puts("\n💡 Add semantic annotations:") + IO.puts(" |> text(\"Hello\", semantic: Semantic.text_buffer(buffer_id: 1))") + IO.puts(" |> rect({100, 40}, semantic: Semantic.button(\"Save\"))") + else + IO.puts("\n🏷️ All Semantic Content:") + + # Group by type + all_elements + |> Enum.group_by(& &1.semantic.type) + |> Enum.sort_by(fn {type, _} -> type end) + |> Enum.each(fn {type, elems} -> + icon = get_type_icon(type) + IO.puts("\n#{icon} #{String.capitalize(to_string(type))} (#{length(elems)}):") + + Enum.each(elems, fn elem -> + desc = format_semantic_element(elem) + IO.puts(" • #{desc}") + end) + end) + end end - defp render_semantic_details(data) do - Enum.each(data.by_type, fn {type, element_ids} -> - icon = get_type_icon(type) - type_name = String.capitalize(to_string(type)) - count = length(element_ids) - - IO.puts("#{icon} #{type_name} (#{count}):") + # ============================================================================ + # Rendering Helpers + # ============================================================================ + + + defp render_graph_semantic(key, data, verbose, _viewport) do + element_count = map_size(data.elements) + + IO.puts("\n 📋 \"#{short_id(key)}\"") + IO.puts(" Elements: #{element_count}") + + if element_count > 0 do + # Group by type for cleaner display + by_type = Enum.group_by(Map.values(data.elements), & &1.semantic.type) - Enum.each(element_ids, fn id -> - elem = Map.get(data.elements, id) - description = format_element_description(elem) - IO.puts(" └── Element ##{id}: #{description}") + Enum.each(by_type, fn {type, elems} -> + icon = get_type_icon(type) - # Show content if it's a text element - if elem.content && String.trim(elem.content) != "" do - content_preview = String.slice(elem.content, 0, 50) - content_preview = if String.length(elem.content) > 50, do: content_preview <> "...", else: content_preview - IO.puts(" Content: #{inspect(content_preview)}") + if verbose do + IO.puts(" #{icon} #{type} (#{length(elems)}):") + Enum.each(elems, fn elem -> + desc = format_element_line(elem) + IO.puts(" • #{desc}") + end) + else + # Compact view - show count only + count = length(elems) + IO.puts(" #{icon} #{type}: #{count} element#{if count > 1, do: "s", else: ""}") end end) + end + end + + defp render_semantic_elements(data) do + data.by_type + |> Enum.sort_by(fn {type, _} -> type end) + |> Enum.each(fn {type, ids} -> + icon = get_type_icon(type) + IO.puts("\n #{icon} #{type} (#{length(ids)}):") - IO.puts("") + Enum.each(ids, fn id -> + elem = Map.get(data.elements, id) + desc = format_semantic_element(elem) + IO.puts(" • #{desc}") + end) end) end - defp format_element_brief(elem) do + # ============================================================================ + # Formatting Helpers + # ============================================================================ + + defp format_element_line(elem) do case elem.semantic.type do - :text_buffer -> "text_buffer: buffer_id \"#{String.slice(elem.semantic.buffer_id, 0, 8)}...\"" - :button -> "button: \"#{elem.semantic.label}\"" - :menu -> "menu: \"#{elem.semantic.name}\"" - :text_input -> "text_input: \"#{elem.semantic.name}\"" - other -> "#{other}: #{inspect(elem.semantic)}" + :text_buffer -> + lines = if elem.content, do: length(String.split(elem.content, "\n")), else: 0 + "Buffer #{short_id(elem.semantic.buffer_id)} (#{lines} lines)" + + :button -> + "\"#{elem.semantic.label}\"" + + :menu -> + "\"#{elem.semantic.name}\"" + + _ -> + inspect(elem.semantic, limit: 1, pretty: false) end end - defp format_element_description(elem) do + + defp format_semantic_element(elem) do + semantic = elem.semantic + + case semantic.type do + :text_buffer -> + id = short_id(semantic.buffer_id) + content_info = if elem.content && elem.content != "" do + lines = length(String.split(elem.content, "\n")) + chars = String.length(elem.content) + " - #{lines} lines, #{chars} chars" + else + " - empty" + end + "#{id}#{content_info}" + + :button -> + "\"#{semantic.label}\"" + + :menu -> + "\"#{semantic.name}\" (#{semantic[:orientation] || :vertical})" + + :text_input -> + name = semantic.name + placeholder = if semantic[:placeholder], do: " (#{semantic.placeholder})", else: "" + "\"#{name}\"#{placeholder}" + + _ -> + # For custom types, show key attributes + attrs = semantic + |> Map.drop([:type]) + |> Enum.map(fn {k, v} -> "#{k}: #{inspect(v, limit: 1)}" end) + |> Enum.join(", ") + + if attrs == "", do: "#{semantic.type}", else: "#{attrs}" + end + end + + defp format_element_for_find(elem) do case elem.semantic.type do - :text_buffer -> - id = String.slice(elem.semantic.buffer_id, 0, 8) <> "..." - preview = if elem.content && String.trim(elem.content) != "" do - " - \"#{String.slice(elem.content, 0, 20)}...\"" + :text_buffer -> + id = short_id(elem.semantic.buffer_id) + preview = if elem.content && elem.content != "" do + first_line = elem.content |> String.split("\n") |> List.first() |> String.slice(0, 40) + " - \"#{first_line}...\"" else - " (empty)" + " - (empty)" end "Buffer #{id}#{preview}" - :button -> "\"#{elem.semantic.label}\"" - :menu -> "\"#{elem.semantic.name}\" menu" - :text_input -> "\"#{elem.semantic.name}\" input" - _other -> "#{inspect(elem.semantic)}" + :button -> + "\"#{elem.semantic.label}\"" + + _ -> + format_semantic_element(elem) + end + end + + # ============================================================================ + # Scene Script Helpers + # ============================================================================ + + defp render_hierarchy_tree(key, data, all_entries, prefix, verbose) do + element_count = map_size(data.elements) + + # Show type counts for the graph + type_info = if element_count > 0 do + semantic_types = Map.keys(data.by_type) |> Enum.join(", ") + " (#{element_count} elements: #{semantic_types})" + else + " (empty)" + end + + IO.puts("#{prefix}📊 #{short_id(key)}#{type_info}") + + # Show children + children = Enum.filter(all_entries, fn {child_key, child_data} -> + child_data.parent == key + end) + + Enum.with_index(children) + |> Enum.each(fn {{child_key, child_data}, idx} -> + is_last = idx == length(children) - 1 + child_prefix = prefix <> if is_last, do: "└── ", else: "├── " + next_prefix = prefix <> if is_last, do: " ", else: "│ " + + render_hierarchy_tree(child_key, child_data, all_entries, child_prefix, verbose) + end) + end + + defp find_matching_elements(entries, opts) do + type = Keyword.get(opts, :type) + role = Keyword.get(opts, :role) + primitive = Keyword.get(opts, :primitive) + + entries + |> Enum.flat_map(fn {graph_key, data} -> + matching_ids = [] + + # Find by semantic type + matching_ids = if type do + Map.get(data.by_type, type, []) ++ matching_ids + else + matching_ids + end + + # Find by role + matching_ids = if role do + Map.get(data.by_role, role, []) ++ matching_ids + else + matching_ids + end + + # Find by primitive type + matching_ids = if primitive do + Map.get(data.by_primitive, primitive, []) ++ matching_ids + else + matching_ids + end + + # If no filters specified, return all elements + matching_ids = if type == nil and role == nil and primitive == nil do + Map.keys(data.elements) + else + matching_ids |> Enum.uniq() + end + + # Convert IDs to full element data + Enum.map(matching_ids, fn id -> + elem = Map.get(data.elements, id) + {graph_key, id, elem} + end) + end) + end + + defp display_found_elements(elements, opts) do + if elements == [] do + IO.puts("No matching elements found") + suggest_available_options(opts) + else + filter_desc = describe_filters(opts) + IO.puts("\n🔍 Found #{length(elements)} element(s)#{filter_desc}:") + + # Group by graph for cleaner display + elements + |> Enum.group_by(fn {graph_key, _, _} -> graph_key end) + |> Enum.each(fn {graph_key, graph_elements} -> + IO.puts("\nGraph \"#{short_id(graph_key)}\":") + + Enum.each(graph_elements, fn {_, id, elem} -> + desc = format_element_detailed(elem) + IO.puts(" • [#{id}] #{desc}") + end) + end) + end + end + + defp describe_filters(opts) do + filters = [] + + filters = if type = Keyword.get(opts, :type) do + ["type: #{type}" | filters] + else + filters + end + + filters = if role = Keyword.get(opts, :role) do + ["role: #{role}" | filters] + else + filters + end + + filters = if primitive = Keyword.get(opts, :primitive) do + name = primitive |> to_string() |> String.split(".") |> List.last() + ["primitive: #{name}" | filters] + else + filters + end + + filters = if in_graph = Keyword.get(opts, :in_graph) do + ["in graph: #{in_graph}" | filters] + else + filters + end + + if filters != [] do + " matching " <> Enum.join(filters, ", ") + else + "" + end + end + + defp format_element_detailed(elem) do + type_info = if primitive_name = elem.type do + name = primitive_name |> to_string() |> String.split(".") |> List.last() + "#{name}" + else + "unknown" + end + + semantic_info = if map_size(elem.semantic) > 0 do + type = Map.get(elem.semantic, :type, "custom") + " (#{type})" + else + "" + end + + content_info = if elem.content do + preview = elem.content |> String.slice(0, 30) + " - \"#{preview}#{if String.length(elem.content) > 30, do: "...", else: ""}\"" + else + "" + end + + "#{type_info}#{semantic_info}#{content_info}" + end + + defp suggest_available_options(opts) do + # This would show what options are actually available + # For now, just show a helpful message + IO.puts("\n💡 Try:") + IO.puts(" • find_element(type: :text_buffer)") + IO.puts(" • find_element(role: :button)") + IO.puts(" • find_element(primitive: Scenic.Primitive.Text)") + IO.puts(" • hierarchy() # Show graph structure") + end + + # ============================================================================ + # Helper Functions + # ============================================================================ + + + + defp show_available_types(entries) do + types = entries + |> Enum.flat_map(fn {_, data} -> Map.keys(data.by_type) end) + |> Enum.uniq() + |> Enum.sort() + + if types != [] do + IO.puts("\nAvailable types: #{Enum.join(types, ", ")}") end end @@ -700,34 +814,24 @@ defmodule Scenic.DevTools do :text_buffer -> "📝" :button -> "🔘" :menu -> "📂" - :text_input -> "📝" + :menu_item -> "📄" + :text_input -> "✏️" + :list -> "📋" + :checkbox -> "☑️" + :radio -> "⭕" + :slider -> "🎚️" + :dropdown -> "📥" _ -> "🏷️" end end - defp format_size({width, height}), do: "#{width}x#{height}" + defp format_size({width, height}), do: "#{width}×#{height}" defp format_size(_), do: "unknown" - defp format_timestamp(timestamp) when is_integer(timestamp) do - # Convert from milliseconds since Unix epoch - datetime = DateTime.from_unix!(timestamp, :millisecond) - Calendar.strftime(datetime, "%Y-%m-%d %H:%M:%S") - end - defp format_timestamp(_), do: "unknown" - - # Helper to find scene info by pid - defp find_scene_by_pid(viewport, pid) do - case Map.get(viewport.scenes_by_pid, pid) do - {id, _parent_id, module} -> {id, module} - nil -> nil - end - end - - # Helper to shorten long IDs - defp short_id(id) when byte_size(id) > 12 do + defp short_id(id) when is_binary(id) and byte_size(id) > 12 do String.slice(id, 0, 8) <> "..." end - defp short_id(id), do: id + defp short_id(id), do: to_string(id) defp get_viewport(viewport) when is_struct(viewport, ViewPort), do: {:ok, viewport} defp get_viewport(name) when is_atom(name) do @@ -737,4 +841,52 @@ defmodule Scenic.DevTools do end end defp get_viewport(pid) when is_pid(pid), do: ViewPort.info(pid) + + # ============================================================================ + # Backwards Compatibility - Deprecated Functions + # ============================================================================ + + @doc false + @deprecated "Scene hierarchy not available through public API" + def scene_tree(_viewport_name \\ :main_viewport) do + IO.puts("⚠️ Scene hierarchy is not available through the public ViewPort interface") + IO.puts("Use inspect_viewport() to see graphs and semantic content instead") + end + + @doc false + @deprecated "Use inspect_viewport/1 instead" + def inspect_app(viewport_name \\ :main_viewport) do + inspect_viewport(viewport: viewport_name) + end + + @doc false + def list_graphs(viewport_name \\ :main_viewport) do + inspect_viewport(viewport: viewport_name, show: :graphs) + end + + @doc false + def semantic_summary(viewport_name \\ :main_viewport) do + inspect_viewport(viewport: viewport_name, show: :semantic) + end + + @doc false + def find_semantic(type, viewport_name \\ :main_viewport) do + find(type, viewport_name) + end + + @doc false + def show_semantic(viewport_name \\ :main_viewport) do + semantic(viewport_name) + end + + @doc false + def inspect_scene(scene_id, viewport_name \\ :main_viewport) do + IO.puts("📌 Note: Use inspect_viewport(graph: \"#{scene_id}\") for better output") + inspect_viewport(viewport: viewport_name, graph: scene_id, verbose: true) + end + + @doc false + def inspect_graph(graph_key, viewport_name \\ :main_viewport) do + inspect_viewport(viewport: viewport_name, graph: graph_key, verbose: true) + end end \ No newline at end of file From abb41d1d20c0f1396078711cdf28d1b8166f1591 Mon Sep 17 00:00:00 2001 From: JediLuke <1879634+JediLuke@users.noreply.github.com> Date: Mon, 14 Jul 2025 17:03:58 -0500 Subject: [PATCH 07/11] wip --- lib/scenic/dev_tools.ex | 944 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 944 insertions(+) diff --git a/lib/scenic/dev_tools.ex b/lib/scenic/dev_tools.ex index 105412c0..e33595ba 100644 --- a/lib/scenic/dev_tools.ex +++ b/lib/scenic/dev_tools.ex @@ -233,6 +233,296 @@ defmodule Scenic.DevTools do end end + @doc """ + Wait for scene hierarchy to be established with retry logic. + + Options: + * `:timeout` - Maximum time to wait in milliseconds (default: 5000) + * `:interval` - Check interval in milliseconds (default: 100) + * `:min_scenes` - Minimum number of scenes expected (default: 2) + """ + def wait_for_scene_hierarchy(viewport_name \\ :main_viewport, opts \\ []) do + timeout = Keyword.get(opts, :timeout, 5000) + interval = Keyword.get(opts, :interval, 100) + min_scenes = Keyword.get(opts, :min_scenes, 2) + + wait_until(timeout, interval, fn -> + data = raw_scene_script(viewport_name) + + cond do + map_size(data) < min_scenes -> + {:error, :insufficient_scenes} + not (Map.has_key?(data, "_root_") and Map.has_key?(data, "_main_")) -> + {:error, :missing_core_scenes} + true -> + # Verify hierarchy is properly established + root = data["_root_"] + main = data["_main_"] + + if root && main && main.parent == "_root_" && length(root.children) > 0 do + {:ok, data} + else + {:error, :hierarchy_not_ready} + end + end + end) + end + + @doc """ + Compare two scene states and show what changed. + """ + def diff_scenes(before_state, after_state) do + %{ + added_graphs: Map.keys(after_state) -- Map.keys(before_state), + removed_graphs: Map.keys(before_state) -- Map.keys(after_state), + modified_graphs: find_modified_graphs(before_state, after_state), + element_changes: diff_elements(before_state, after_state), + hierarchy_changes: diff_hierarchy(before_state, after_state) + } + end + + @doc """ + Track scene changes during an operation. + """ + def track_changes(viewport_name \\ :main_viewport, fun) when is_function(fun, 0) do + before = raw_scene_script(viewport_name) + before_time = System.monotonic_time(:microsecond) + + result = fun.() + + after_time = System.monotonic_time(:microsecond) + after_state = raw_scene_script(viewport_name) + + changes = diff_scenes(before, after_state) + duration_us = after_time - before_time + + {result, Map.put(changes, :duration_us, duration_us)} + end + + @doc """ + Generate an ASCII art representation of the scene hierarchy. + """ + def visualize_hierarchy(viewport_name \\ :main_viewport) do + data = raw_scene_script(viewport_name) + + IO.puts("\n╔═══════════════════════════════════╗") + IO.puts("║ Scene Hierarchy Diagram ║") + IO.puts("╚═══════════════════════════════════╝\n") + + if map_size(data) == 0 do + IO.puts(" (No scenes found)") + else + render_hierarchy_tree(data, "_root_", "", true, 0) + end + end + + @doc """ + Show a heat map of element density across scenes. + """ + def element_heatmap(viewport_name \\ :main_viewport) do + data = raw_scene_script(viewport_name) + + heatmap = data + |> Enum.map(fn {key, scene} -> + density = map_size(scene.elements) + bar = String.duplicate("█", min(density, 50)) + {key, density, bar} + end) + |> Enum.sort_by(fn {_, density, _} -> -density end) + + IO.puts("\n📊 Element Density Heatmap:") + IO.puts("════════════════════════════") + Enum.each(heatmap, fn {key, density, bar} -> + short_key = short_id(key) + padding = String.duplicate(" ", max(0, 20 - String.length(short_key))) + IO.puts("#{short_key}#{padding} [#{String.pad_leading(Integer.to_string(density), 3)}] #{bar}") + end) + IO.puts("") + end + + @doc """ + Run comprehensive diagnostics when scene structure is unexpected. + """ + def diagnose_scene_issues(viewport_name \\ :main_viewport) do + IO.puts("\n🏥 Scene Diagnostics Report") + IO.puts("═══════════════════════════") + + # Check viewport + case get_viewport(viewport_name) do + {:ok, vp} -> + IO.puts("✓ Viewport accessible: #{inspect(viewport_name)}") + diagnose_viewport(vp) + {:error, reason} -> + IO.puts("✗ Viewport error: #{reason}") + IO.puts("\n⚠️ Cannot access viewport!") + IO.puts(" Possible causes:") + IO.puts(" - Application not started") + IO.puts(" - Wrong viewport name (try :main_viewport)") + IO.puts(" - ViewPort process crashed") + end + + # Check Scenic processes + check_scenic_processes() + + # Suggest fixes + suggest_remedies() + end + + # Private helper functions + + defp wait_until(timeout, _interval, _fun) when timeout <= 0 do + {:error, :timeout} + end + + defp wait_until(timeout, interval, fun) do + case fun.() do + {:ok, result} -> + {:ok, result} + {:error, _reason} -> + Process.sleep(interval) + wait_until(timeout - interval, interval, fun) + end + end + + defp find_modified_graphs(before_state, after_state) do + common_keys = Map.keys(before_state) -- (Map.keys(before_state) -- Map.keys(after_state)) + + Enum.filter(common_keys, fn key -> + before_data = Map.get(before_state, key) + after_data = Map.get(after_state, key) + + before_data != after_data + end) + end + + defp diff_elements(before_state, after_state) do + %{ + total_before: count_all_elements(before_state), + total_after: count_all_elements(after_state), + by_graph: element_changes_by_graph(before_state, after_state) + } + end + + defp diff_hierarchy(before_state, after_state) do + before_structure = extract_hierarchy_structure(before_state) + after_structure = extract_hierarchy_structure(after_state) + + %{ + parent_changes: find_parent_changes(before_structure, after_structure), + depth_changes: find_depth_changes(before_structure, after_structure) + } + end + + defp count_all_elements(scene_data) do + scene_data + |> Map.values() + |> Enum.map(fn scene -> map_size(scene.elements) end) + |> Enum.sum() + end + + defp element_changes_by_graph(before_state, after_state) do + all_keys = Map.keys(before_state) ++ Map.keys(after_state) |> Enum.uniq() + + Map.new(all_keys, fn key -> + before_count = get_in(before_state, [key, :elements]) |> map_size_safe() + after_count = get_in(after_state, [key, :elements]) |> map_size_safe() + + {key, %{before: before_count, after: after_count, change: after_count - before_count}} + end) + end + + defp map_size_safe(nil), do: 0 + defp map_size_safe(map) when is_map(map), do: map_size(map) + defp map_size_safe(_), do: 0 + + defp extract_hierarchy_structure(scene_data) do + Map.new(scene_data, fn {key, scene} -> + {key, %{parent: scene.parent, depth: scene.depth, children: scene.children}} + end) + end + + defp find_parent_changes(before_structure, after_structure) do + Enum.reduce(after_structure, [], fn {key, after_info}, changes -> + case Map.get(before_structure, key) do + nil -> changes + before_info -> + if before_info.parent != after_info.parent do + [{key, %{from: before_info.parent, to: after_info.parent}} | changes] + else + changes + end + end + end) + end + + defp find_depth_changes(before_structure, after_structure) do + Enum.reduce(after_structure, [], fn {key, after_info}, changes -> + case Map.get(before_structure, key) do + nil -> changes + before_info -> + if before_info.depth != after_info.depth do + [{key, %{from: before_info.depth, to: after_info.depth}} | changes] + else + changes + end + end + end) + end + + defp diagnose_viewport(viewport) do + # Check scene script table + script_count = case :ets.info(viewport.scene_script_table, :size) do + :undefined -> 0 + count -> count + end + IO.puts(" Scene scripts: #{script_count}") + + # Check semantic table + semantic_count = case :ets.info(viewport.semantic_table, :size) do + :undefined -> 0 + count -> count + end + IO.puts(" Semantic entries: #{semantic_count}") + + # Check for common issues + if script_count == 0 do + IO.puts("\n⚠️ No scene scripts found!") + IO.puts(" Possible causes:") + IO.puts(" - Application just started (try waiting)") + IO.puts(" - Scene not properly initialized") + IO.puts(" - ViewPort enhancement not enabled") + end + end + + defp check_scenic_processes() do + IO.puts("\n📋 Process Status:") + + # Check if Scenic application is running + case Application.get_application(Scenic.ViewPort) do + :scenic -> + IO.puts(" ✓ Scenic application running") + _ -> + IO.puts(" ✗ Scenic application not detected") + end + + # Check for viewport processes + viewport_count = Process.registered() + |> Enum.filter(&(Atom.to_string(&1) =~ "viewport")) + |> length() + + IO.puts(" ViewPort processes: #{viewport_count}") + end + + defp suggest_remedies() do + IO.puts("\n💡 Suggested Actions:") + IO.puts(" 1. Wait for scene initialization:") + IO.puts(" Scenic.DevTools.wait_for_scene_hierarchy()") + IO.puts(" 2. Check viewport names:") + IO.puts(" Scenic.ViewPort.list()") + IO.puts(" 3. Force scene refresh:") + IO.puts(" Scenic.ViewPort.reset(:main_viewport)") + end + @doc """ Show the hierarchical structure of graphs. @@ -266,6 +556,94 @@ defmodule Scenic.DevTools do end end + @doc """ + Introspect the scene - a high-level view showing your app as scenes and components. + + This is the top-level developer interface that presents your application + in terms of scenes, components, and their relationships rather than + low-level graphs and primitives. + + ## Examples + + # Overview of your entire application + iex> introspect() + + # Dive into a specific scene/component + iex> introspect("main_editor") + + # Show with detailed component info + iex> introspect(detailed: true) + """ + def introspect(opts \\ []) + def introspect(scene_name) when is_binary(scene_name) do + introspect(scene: scene_name) + end + def introspect(opts) when is_list(opts) do + viewport_name = Keyword.get(opts, :viewport, :main_viewport) + scene_filter = Keyword.get(opts, :scene) + detailed = Keyword.get(opts, :detailed, false) + + with {:ok, viewport} <- get_viewport(viewport_name) do + IO.puts("\n╔══════ Scene Introspection ══════╗") + IO.puts("║ Application: #{format_app_name(viewport_name)}") + IO.puts("║ ViewPort: #{format_size(viewport.size)}") + IO.puts("╚═════════════════════════════════╝\n") + + scene_analysis = analyze_scenes(viewport) + + if scene_filter do + show_scene_detail(scene_analysis, scene_filter, detailed) + else + show_application_overview(scene_analysis, detailed) + end + + show_introspection_tips() + :ok + else + error -> + IO.puts("❌ Error: #{inspect(error)}") + :error + end + end + + @doc """ + Explore a specific component or scene interactively. + + Shows the component breakdown, its role in the application, + and what interactive elements it contains. + """ + def explore(component_name, viewport_name \\ :main_viewport) do + with {:ok, viewport} <- get_viewport(viewport_name) do + scene_analysis = analyze_scenes(viewport) + component = find_component_by_name(scene_analysis, component_name) + + if component do + show_component_explorer(component, scene_analysis) + else + IO.puts("🔍 Component '#{component_name}' not found") + suggest_available_components(scene_analysis) + end + :ok + end + end + + @doc """ + Show the application's component architecture. + + Displays how your application is structured in terms of + reusable components and their relationships. + """ + def architecture(viewport_name \\ :main_viewport) do + with {:ok, viewport} <- get_viewport(viewport_name) do + IO.puts("\n🏛️ Application Architecture") + IO.puts("══════════════════════════") + + scene_analysis = analyze_scenes(viewport) + show_architecture_overview(scene_analysis) + :ok + end + end + @doc """ Find elements across all graphs with advanced filtering. @@ -623,6 +1001,282 @@ defmodule Scenic.DevTools do end end + # ============================================================================ + # Scene Analysis & Introspection + # ============================================================================ + + # Analyze the viewport and convert raw graph data into scene/component concepts + defp analyze_scenes(viewport) do + script_entries = :ets.tab2list(viewport.scene_script_table) + semantic_entries = :ets.tab2list(viewport.semantic_table) + + # Combine script and semantic data + combined_data = merge_scene_data(script_entries, semantic_entries) + + # Identify scene types and roles + scenes = identify_scenes(combined_data) + + # Build component relationships + components = extract_components(scenes) + + %{ + raw_graphs: combined_data, + scenes: scenes, + components: components, + app_structure: build_app_structure(scenes, components) + } + end + + defp merge_scene_data(script_entries, semantic_entries) do + # Create a map of graph_key -> combined data + script_map = Map.new(script_entries) + semantic_map = Map.new(semantic_entries) + + # Start with script data (which has hierarchy) and enhance with semantic data + script_map + |> Enum.map(fn {key, script_data} -> + semantic_data = Map.get(semantic_map, key, %{elements: %{}, by_type: %{}}) + + combined = Map.merge(script_data, %{ + semantic_elements: semantic_data.elements, + semantic_by_type: Map.get(semantic_data, :by_type, %{}) + }) + + {key, combined} + end) + |> Map.new() + end + + defp identify_scenes(combined_data) do + combined_data + |> Enum.map(fn {key, data} -> + scene_type = determine_scene_type(key, data) + scene_name = determine_scene_name(key, data, scene_type) + + %{ + graph_key: key, + scene_name: scene_name, + scene_type: scene_type, + parent: data.parent, + children: data.children, + elements: data.elements, + semantic_elements: data.semantic_elements, + interactive_elements: count_interactive_elements(data), + purpose: determine_scene_purpose(data), + depth: data.depth + } + end) + |> Enum.sort_by(& &1.depth) + end + + defp determine_scene_type(key, data) do + cond do + key == "_root_" -> :root + key == "_main_" -> :main + String.starts_with?(key, "_") -> :system + map_size(data.elements) == 0 -> :container + has_text_editing?(data) -> :editor + has_buttons_or_menus?(data) -> :interface + has_display_content?(data) -> :display + true -> :component + end + end + + defp determine_scene_name(key, data, scene_type) do + case scene_type do + :root -> "Application Root" + :main -> "Main Scene" + :editor -> extract_editor_name(data) || "Text Editor" + :interface -> extract_interface_name(data) || "UI Controls" + :display -> "Display Area" + :container -> "Layout Container" + :component -> extract_component_name(data) || short_id(key) + :system -> "System (#{short_id(key)})" + end + end + + defp extract_components(scenes) do + scenes + |> Enum.filter(fn scene -> scene.scene_type not in [:root, :main, :system] end) + |> Enum.map(fn scene -> + %{ + name: scene.scene_name, + type: scene.scene_type, + graph_key: scene.graph_key, + capabilities: extract_capabilities(scene), + interactive_count: scene.interactive_elements, + purpose: scene.purpose + } + end) + end + + defp build_app_structure(scenes, components) do + root_scene = Enum.find(scenes, & &1.scene_type == :root) + main_scene = Enum.find(scenes, & &1.scene_type == :main) + + %{ + entry_point: root_scene, + main_interface: main_scene, + component_count: length(components), + scene_depth: Enum.map(scenes, & &1.depth) |> Enum.max(fn -> 0 end), + interaction_points: Enum.sum(Enum.map(scenes, & &1.interactive_elements)) + } + end + + # ============================================================================ + # Scene Display Functions + # ============================================================================ + + defp show_application_overview(analysis, detailed) do + IO.puts("🎭 Application Scenes") + IO.puts("━━━━━━━━━━━━━━━━━━━━") + + # Show the scene hierarchy with purpose and capabilities + show_scene_tree(analysis.scenes, detailed) + + IO.puts("\n📊 Application Summary") + IO.puts("━━━━━━━━━━━━━━━━━━━━━━") + show_app_summary(analysis) + + if detailed do + IO.puts("\n🧩 Components") + IO.puts("━━━━━━━━━━━━━") + show_components_summary(analysis.components) + end + end + + defp show_scene_tree(scenes, detailed) do + # Find root and show hierarchy + roots = Enum.filter(scenes, & &1.parent == nil) + + Enum.each(roots, fn root -> + render_scene_tree(root, scenes, "", detailed) + end) + end + + defp render_scene_tree(scene, all_scenes, prefix, detailed) do + icon = get_scene_icon(scene.scene_type) + name = scene.scene_name + + info = if scene.interactive_elements > 0 do + " (#{scene.interactive_elements} interactive)" + else + "" + end + + purpose = if detailed and scene.purpose do + " - #{scene.purpose}" + else + "" + end + + IO.puts("#{prefix}#{icon} #{name}#{info}#{purpose}") + + # Show children + children = Enum.filter(all_scenes, & &1.parent == scene.graph_key) + + Enum.with_index(children) + |> Enum.each(fn {child, idx} -> + is_last = idx == length(children) - 1 + child_prefix = prefix <> if is_last, do: "└── ", else: "├── " + + render_scene_tree(child, all_scenes, child_prefix, detailed) + end) + end + + defp show_app_summary(analysis) do + structure = analysis.app_structure + + IO.puts(" Scenes: #{length(analysis.scenes)}") + IO.puts(" Components: #{structure.component_count}") + IO.puts(" Scene Depth: #{structure.scene_depth}") + IO.puts(" Interactive Elements: #{structure.interaction_points}") + + # Show breakdown by scene type + type_counts = analysis.scenes + |> Enum.group_by(& &1.scene_type) + |> Enum.map(fn {type, scenes} -> {type, length(scenes)} end) + |> Enum.sort_by(fn {_, count} -> -count end) + + if length(type_counts) > 1 do + IO.puts("\n Scene Types:") + Enum.each(type_counts, fn {type, count} -> + icon = get_scene_icon(type) + IO.puts(" #{icon} #{format_scene_type(type)}: #{count}") + end) + end + end + + defp show_components_summary(components) do + if components == [] do + IO.puts(" No interactive components identified") + else + Enum.each(components, fn component -> + icon = get_scene_icon(component.type) + capabilities = Enum.join(component.capabilities, ", ") + IO.puts(" #{icon} #{component.name}") + if capabilities != "" do + IO.puts(" Capabilities: #{capabilities}") + end + if component.interactive_count > 0 do + IO.puts(" Interactive: #{component.interactive_count} elements") + end + end) + end + end + + defp show_scene_detail(analysis, scene_filter, detailed) do + scene = Enum.find(analysis.scenes, fn s -> + String.contains?(String.downcase(s.scene_name), String.downcase(scene_filter)) or + String.contains?(s.graph_key, scene_filter) + end) + + if scene do + show_component_explorer(scene, analysis) + else + IO.puts("🔍 Scene '#{scene_filter}' not found") + IO.puts("\nAvailable scenes:") + Enum.each(analysis.scenes, fn s -> + icon = get_scene_icon(s.scene_type) + IO.puts(" #{icon} #{s.scene_name} (#{s.graph_key})") + end) + end + end + + defp show_component_explorer(component, analysis) do + IO.puts("🔍 Exploring: #{component.scene_name}") + IO.puts("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━") + + IO.puts("📋 Component Details:") + IO.puts(" Type: #{format_scene_type(component.scene_type)}") + IO.puts(" Graph Key: #{component.graph_key}") + IO.puts(" Purpose: #{component.purpose || "General component"}") + + if component.interactive_elements > 0 do + IO.puts("\n🎯 Interactive Elements:") + show_interactive_breakdown(component) + end + + show_component_relationships(component, analysis) + + if map_size(component.elements) > 0 do + IO.puts("\n🔧 Technical Details:") + IO.puts(" Total Elements: #{map_size(component.elements)}") + show_element_breakdown(component) + end + end + + defp show_architecture_overview(analysis) do + IO.puts("\n📐 Structure Overview:") + show_scene_tree(analysis.scenes, false) + + IO.puts("\n🔗 Component Relationships:") + show_component_relationships_overview(analysis) + + IO.puts("\n🎯 Interaction Design:") + show_interaction_analysis(analysis) + end + # ============================================================================ # Scene Script Helpers # ============================================================================ @@ -792,6 +1446,296 @@ defmodule Scenic.DevTools do IO.puts(" • hierarchy() # Show graph structure") end + # ============================================================================ + # Scene Analysis Helper Functions + # ============================================================================ + + # Determine what this scene/component is for based on its elements + defp determine_scene_purpose(data) do + cond do + has_text_editing?(data) -> "Text editing and content creation" + has_form_elements?(data) -> "User input and form interaction" + has_navigation?(data) -> "Application navigation" + has_display_content?(data) -> "Content display and visualization" + has_buttons_or_menus?(data) -> "User interface controls" + map_size(data.elements) == 0 -> "Layout and structure" + true -> "Component functionality" + end + end + + # Check what capabilities a scene/component has + defp extract_capabilities(scene) do + capabilities = [] + + capabilities = if has_text_editing?(scene) do + ["text editing" | capabilities] + else + capabilities + end + + capabilities = if has_buttons_or_menus?(scene) do + ["interactive controls" | capabilities] + else + capabilities + end + + capabilities = if has_form_elements?(scene) do + ["data input" | capabilities] + else + capabilities + end + + capabilities = if has_display_content?(scene) do + ["content display" | capabilities] + else + capabilities + end + + if capabilities == [] do + ["layout"] + else + capabilities + end + end + + # Scene type detection helpers + defp has_text_editing?(data) do + Map.get(data, :semantic_by_type, %{}) + |> Map.has_key?(:text_buffer) + end + + defp has_buttons_or_menus?(data) do + by_type = Map.get(data, :semantic_by_type, %{}) + Map.has_key?(by_type, :button) or Map.has_key?(by_type, :menu) + end + + defp has_form_elements?(data) do + by_type = Map.get(data, :semantic_by_type, %{}) + Map.has_key?(by_type, :text_input) or Map.has_key?(by_type, :checkbox) + end + + defp has_navigation?(data) do + by_type = Map.get(data, :semantic_by_type, %{}) + Map.has_key?(by_type, :menu) or Map.has_key?(by_type, :navigation) + end + + defp has_display_content?(data) do + by_primitive = Map.get(data, :by_primitive, %{}) + + text_elements = Map.get(by_primitive, Scenic.Primitive.Text, []) + other_visual = Map.get(by_primitive, Scenic.Primitive.Rectangle, []) ++ + Map.get(by_primitive, Scenic.Primitive.Circle, []) + + length(text_elements) > 0 or length(other_visual) > 0 + end + + defp count_interactive_elements(data) do + by_type = Map.get(data, :semantic_by_type, %{}) + + button_count = length(Map.get(by_type, :button, [])) + menu_count = length(Map.get(by_type, :menu, [])) + input_count = length(Map.get(by_type, :text_input, [])) + buffer_count = length(Map.get(by_type, :text_buffer, [])) + + button_count + menu_count + input_count + buffer_count + end + + # Name extraction helpers + defp extract_editor_name(data) do + # Try to get buffer name or file path from semantic data + text_buffers = Map.get(data, :semantic_elements, %{}) + |> Map.values() + |> Enum.filter(fn elem -> + get_in(elem, [:semantic, :type]) == :text_buffer + end) + + case text_buffers do + [buffer | _] -> + file_path = get_in(buffer, [:semantic, :file_path]) + if file_path do + "Editor (#{Path.basename(file_path)})" + else + nil + end + _ -> nil + end + end + + defp extract_interface_name(data) do + # Try to identify the interface based on button labels + buttons = Map.get(data, :semantic_elements, %{}) + |> Map.values() + |> Enum.filter(fn elem -> + get_in(elem, [:semantic, :type]) == :button + end) + + if length(buttons) > 2 do + "Toolbar" + else + nil + end + end + + defp extract_component_name(data) do + # Try to extract a meaningful name from semantic data + elements = Map.get(data, :semantic_elements, %{}) + + case Map.values(elements) do + [elem | _] -> + semantic = Map.get(elem, :semantic, %{}) + Map.get(semantic, :name) || Map.get(semantic, :label) + _ -> nil + end + end + + # Display helpers for introspection + defp get_scene_icon(scene_type) do + case scene_type do + :root -> "🏠" + :main -> "🖥️" + :editor -> "📝" + :interface -> "🎛️" + :display -> "📺" + :container -> "📦" + :component -> "🧩" + :system -> "⚙️" + _ -> "❓" + end + end + + defp format_scene_type(scene_type) do + case scene_type do + :root -> "Root Scene" + :main -> "Main Interface" + :editor -> "Text Editor" + :interface -> "UI Controls" + :display -> "Display Component" + :container -> "Layout Container" + :component -> "Component" + :system -> "System Component" + _ -> to_string(scene_type) + end + end + + defp format_app_name(viewport_name) do + case viewport_name do + :main_viewport -> "Scenic Application" + name -> name |> to_string() |> String.replace("_", " ") |> String.split() |> Enum.map(&String.capitalize/1) |> Enum.join(" ") + end + end + + defp show_introspection_tips do + IO.puts("\n💡 Introspection Commands:") + IO.puts(" • introspect(detailed: true) # Detailed view") + IO.puts(" • introspect(\"editor\") # Focus on specific scene") + IO.puts(" • explore(\"Text Editor\") # Interactive exploration") + IO.puts(" • architecture() # Component relationships") + end + + # Stubs for functions that need implementation + defp find_component_by_name(analysis, component_name) do + Enum.find(analysis.scenes, fn scene -> + String.contains?(String.downcase(scene.scene_name), String.downcase(component_name)) + end) + end + + defp suggest_available_components(analysis) do + IO.puts("\nAvailable components:") + Enum.each(analysis.scenes, fn scene -> + icon = get_scene_icon(scene.scene_type) + IO.puts(" #{icon} #{scene.scene_name}") + end) + end + + defp show_interactive_breakdown(component) do + if map_size(component.semantic_elements) > 0 do + component.semantic_elements + |> Map.values() + |> Enum.filter(fn elem -> + type = get_in(elem, [:semantic, :type]) + type in [:button, :text_buffer, :text_input, :menu] + end) + |> Enum.each(fn elem -> + type_icon = get_type_icon(elem.semantic.type) + name = get_in(elem, [:semantic, :label]) || + get_in(elem, [:semantic, :name]) || + "Unnamed #{elem.semantic.type}" + IO.puts(" #{type_icon} #{name}") + end) + else + IO.puts(" (No semantic annotations found)") + end + end + + defp show_component_relationships(component, analysis) do + # Show parent and children + if component.parent do + parent = Enum.find(analysis.scenes, & &1.graph_key == component.parent) + if parent do + IO.puts("\n🔗 Relationships:") + IO.puts(" Parent: #{get_scene_icon(parent.scene_type)} #{parent.scene_name}") + end + end + + children = Enum.filter(analysis.scenes, & &1.parent == component.graph_key) + if children != [] do + if component.parent == nil do + IO.puts("\n🔗 Relationships:") + end + IO.puts(" Children:") + Enum.each(children, fn child -> + IO.puts(" #{get_scene_icon(child.scene_type)} #{child.scene_name}") + end) + end + end + + defp show_element_breakdown(component) do + by_primitive = component.by_primitive || %{} + + if map_size(by_primitive) > 0 do + by_primitive + |> Enum.sort_by(fn {_, ids} -> -length(ids) end) + |> Enum.each(fn {primitive_type, ids} -> + name = primitive_type |> to_string() |> String.split(".") |> List.last() + IO.puts(" #{name}: #{length(ids)}") + end) + end + end + + defp show_component_relationships_overview(analysis) do + # Show how components connect to each other + roots = Enum.filter(analysis.scenes, & &1.parent == nil) + + Enum.each(roots, fn root -> + children = Enum.filter(analysis.scenes, & &1.parent == root.graph_key) + if children != [] do + IO.puts(" #{get_scene_icon(root.scene_type)} #{root.scene_name}") + Enum.each(children, fn child -> + IO.puts(" └── #{get_scene_icon(child.scene_type)} #{child.scene_name}") + end) + end + end) + end + + defp show_interaction_analysis(analysis) do + total_interactive = Enum.sum(Enum.map(analysis.scenes, & &1.interactive_elements)) + + IO.puts(" Total Interactive Elements: #{total_interactive}") + + # Show which scenes have the most interaction + interactive_scenes = analysis.scenes + |> Enum.filter(& &1.interactive_elements > 0) + |> Enum.sort_by(& -&1.interactive_elements) + + if interactive_scenes != [] do + IO.puts(" Most Interactive:") + Enum.take(interactive_scenes, 3) + |> Enum.each(fn scene -> + IO.puts(" #{get_scene_icon(scene.scene_type)} #{scene.scene_name}: #{scene.interactive_elements}") + end) + end + end + # ============================================================================ # Helper Functions # ============================================================================ From 5049ae219b453b18bcd3933329618c1d50659bff Mon Sep 17 00:00:00 2001 From: JediLuke <1879634+JediLuke@users.noreply.github.com> Date: Mon, 14 Jul 2025 17:04:02 -0500 Subject: [PATCH 08/11] wip2 --- lib/scenic/view_port.ex | 275 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 273 insertions(+), 2 deletions(-) diff --git a/lib/scenic/view_port.ex b/lib/scenic/view_port.ex index d645e6ec..943f8c21 100644 --- a/lib/scenic/view_port.ex +++ b/lib/scenic/view_port.ex @@ -115,6 +115,7 @@ defmodule Scenic.ViewPort do # name_table: reference, script_table: reference, semantic_table: reference, + scene_script_table: reference, size: {number, number} } defstruct name: nil, @@ -122,6 +123,7 @@ defmodule Scenic.ViewPort do # name_table: nil, script_table: nil, semantic_table: nil, + scene_script_table: nil, size: nil @viewports :scenic_viewports @@ -535,7 +537,7 @@ defmodule Scenic.ViewPort do graph :: Graph.t(), opts :: Keyword.t() ) :: {:ok, name :: any} | {:error, atom} - def put_graph(%ViewPort{pid: pid, semantic_table: semantic_table} = viewport, name, %Graph{} = graph, opts \\ []) do + def put_graph(%ViewPort{pid: pid, semantic_table: semantic_table, scene_script_table: scene_script_table} = viewport, name, %Graph{} = graph, opts \\ []) do opts = opts |> Enum.into([]) @@ -563,6 +565,13 @@ defmodule Scenic.ViewPort do # Build and store semantic information semantic_info = build_semantic_info(graph, name) true = :ets.insert(semantic_table, {name, semantic_info}) + + # Build and store enhanced scene script information + scene_script_info = build_scene_script_info(graph, name, script, %{}) + true = :ets.insert(scene_script_table, {name, scene_script_info}) + + # Recompute hierarchy for all graphs after each update + recompute_scene_script_hierarchy(scene_script_table) # send the input list to the viewport GenServer.cast(pid, {:input_list, input_list, name, owner}) @@ -732,6 +741,7 @@ defmodule Scenic.ViewPort do # name_table = :ets.new(:_vp_name_table_, [:protected]) script_table = :ets.new(:_vp_script_table_, [:public, {:read_concurrency, true}]) semantic_table = :ets.new(:_vp_semantic_table_, [:public, {:read_concurrency, true}]) + scene_script_table = :ets.new(:_vp_scene_script_table_, [:public, {:read_concurrency, true}]) state = %{ # simple metadata about the ViewPort @@ -770,6 +780,7 @@ defmodule Scenic.ViewPort do # finished scripts to the VP for writing. script_table: script_table, semantic_table: semantic_table, + scene_script_table: scene_script_table, # state related to input from drivers to scenes # input lists are generated when a scene pushes a graph. Primitives @@ -1413,6 +1424,7 @@ defmodule Scenic.ViewPort do # name_table: name_table, script_table: script_table, semantic_table: semantic_table, + scene_script_table: scene_script_table, size: size }) do %ViewPort{ @@ -1421,6 +1433,7 @@ defmodule Scenic.ViewPort do # name_table: name_table, script_table: script_table, semantic_table: semantic_table, + scene_script_table: scene_script_table, size: size } end @@ -1467,7 +1480,7 @@ defmodule Scenic.ViewPort do defp internal_put_graph( %Graph{} = graph, name, - %{input_lists: ils, script_table: script_table, semantic_table: semantic_table} = state + %{input_lists: ils, script_table: script_table, semantic_table: semantic_table, scene_script_table: scene_script_table} = state ) do state = with {:ok, script} <- GraphCompiler.compile(graph), @@ -1486,6 +1499,13 @@ defmodule Scenic.ViewPort do semantic_info = build_semantic_info(graph, name) true = :ets.insert(semantic_table, {name, semantic_info}) + # Build and store enhanced scene script information + scene_script_info = build_scene_script_info(graph, name, script, state) + true = :ets.insert(scene_script_table, {name, scene_script_info}) + + # Recompute hierarchy for all graphs after each update + recompute_scene_script_hierarchy(scene_script_table) + :ok end @@ -2107,6 +2127,257 @@ defmodule Scenic.ViewPort do } end + # Build enhanced scene script information with hierarchy and metadata + defp build_scene_script_info(graph, graph_key, script, state) do + # Extract all elements (not just semantic ones) + elements = extract_all_elements(graph) + + # Extract script references for hierarchy + children = extract_script_references(script) + + # Debug output (removed for production) + + # Build enhanced element data + enhanced_elements = enhance_elements(elements, graph) + + %{ + # === HIERARCHY INFORMATION === + graph_key: graph_key, + children: children, + parent: nil, # Will be computed during hierarchy pass + depth: 0, # Will be computed during hierarchy pass + render_order: 0, # Will be computed during hierarchy pass + + # === METADATA === + timestamp: System.system_time(:millisecond), + owner_pid: determine_owner_pid(state), + + # === VISUAL INFORMATION === + transforms: extract_graph_transforms(script), + bounds: %{x: 0, y: 0, w: 0, h: 0}, # Will be computed from primitives + + # === ELEMENTS (Enhanced from current semantic system) === + elements: enhanced_elements, + + # === FAST LOOKUPS === + by_type: group_elements_by_semantic_type(enhanced_elements), + by_role: group_elements_by_role(enhanced_elements), + by_primitive: group_elements_by_primitive_type(enhanced_elements) + } + end + + # Extract all primitives, not just those with semantic data + defp extract_all_elements(graph) do + graph.primitives + |> Enum.reduce(%{}, fn {id, primitive}, acc -> + element_info = %{ + id: id, + type: primitive.module, + primitive_data: primitive.data, + transforms: primitive.transforms, + + # Semantic data (if present) + semantic: extract_semantic_data(primitive), + + # Content (for text primitives) + content: extract_content(primitive), + + # Computed properties for automation + clickable: is_clickable_primitive(primitive), + visible: true, # Will be computed based on transforms/clips + text_selectable: is_text_selectable(primitive) + } + Map.put(acc, id, element_info) + end) + end + + # Extract semantic data from primitive options + defp extract_semantic_data(primitive) do + case Map.get(primitive, :opts) do + nil -> %{} + opts when is_list(opts) -> Keyword.get(opts, :semantic, %{}) + _ -> %{} + end + end + + # Extract script references from compiled script + defp extract_script_references(script) when is_list(script) do + references = script + |> Enum.filter(fn + {:script, _child_key} -> true + _ -> false + end) + |> Enum.map(fn {:script, key} -> key end) + |> Enum.uniq() + + # Debug output (removed for production) + + references + end + defp extract_script_references(_), do: [] + + # Extract graph-level transforms from script + defp extract_graph_transforms(script) when is_list(script) do + script + |> Enum.filter(fn + {:push_transform, _} -> true + {:translate, _} -> true + {:scale, _} -> true + {:rotate, _} -> true + _ -> false + end) + end + defp extract_graph_transforms(_), do: [] + + # Enhance elements with computed properties + defp enhance_elements(elements, _graph) do + # For now, just return elements as-is + # TODO: Add bounds calculation, visibility computation, etc. + elements + end + + # Determine if primitive is clickable + defp is_clickable_primitive(%{module: Scenic.Primitive.RoundedRectangle}), do: true + defp is_clickable_primitive(%{module: Scenic.Primitive.Rectangle}), do: true + defp is_clickable_primitive(%{module: Scenic.Primitive.Circle}), do: true + defp is_clickable_primitive(%{module: Scenic.Primitive.Ellipse}), do: true + defp is_clickable_primitive(primitive) do + # Check if primitive has semantic role that suggests clickability + semantic = extract_semantic_data(primitive) + case Map.get(semantic, :role) do + :button -> true + :link -> true + _ -> false + end + end + + # Determine if primitive contains selectable text + defp is_text_selectable(%{module: Scenic.Primitive.Text}), do: true + defp is_text_selectable(primitive) do + semantic = extract_semantic_data(primitive) + Map.get(semantic, :type) == :text_buffer + end + + # Group elements by accessibility role + defp group_elements_by_role(elements) do + elements + |> Enum.reduce(%{}, fn {id, element}, acc -> + if role = get_in(element, [:semantic, :role]) do + Map.update(acc, role, [id], &[id | &1]) + else + acc + end + end) + end + + # Group elements by primitive type + defp group_elements_by_primitive_type(elements) do + elements + |> Enum.reduce(%{}, fn {id, element}, acc -> + primitive_type = element.type + Map.update(acc, primitive_type, [id], &[id | &1]) + end) + end + + # Determine owner PID from state + defp determine_owner_pid(_state) do + # For now, return nil - this would need access to the owner info + # from the graph insertion context + nil + end + + # Recompute hierarchy relationships for all scene scripts + defp recompute_scene_script_hierarchy(scene_script_table) do + # Get all current scene script entries + entries = :ets.tab2list(scene_script_table) + + # Build parent/child relationships and compute depths + updated_entries = compute_hierarchy_relationships(entries) + + # Update all entries with new hierarchy information + Enum.each(updated_entries, fn {key, updated_data} -> + :ets.insert(scene_script_table, {key, updated_data}) + end) + end + + # Compute parent/child relationships and depths for all scene scripts + defp compute_hierarchy_relationships(entries) do + # Create a map for easier lookups + data_map = Map.new(entries) + + # Build parent relationships by finding who references each graph + entries_with_parents = Enum.map(entries, fn {key, data} -> + parent = find_parent_graph(key, data_map) + updated_data = Map.put(data, :parent, parent) + {key, updated_data} + end) + + # Compute depths starting from root nodes + entries_with_depths = compute_depths(entries_with_parents) + + entries_with_depths + end + + # Find which graph references this one as a child + defp find_parent_graph(target_key, data_map) do + Enum.find_value(data_map, fn {graph_key, graph_data} -> + if target_key in graph_data.children do + graph_key + else + nil + end + end) + end + + # Compute depth for each graph based on its position in the hierarchy + defp compute_depths(entries_with_parents) do + data_map = Map.new(entries_with_parents) + + # Find root graphs (no parent) + roots = Enum.filter(entries_with_parents, fn {_key, data} -> + data.parent == nil + end) + + # Assign depths starting from roots + depth_assignments = compute_depths_recursive(roots, data_map, %{}, 0) + + # Apply depth assignments to all entries + Enum.map(entries_with_parents, fn {key, data} -> + depth = Map.get(depth_assignments, key, 0) + updated_data = Map.put(data, :depth, depth) + {key, updated_data} + end) + end + + # Recursively compute depths for the hierarchy + defp compute_depths_recursive(nodes, data_map, depth_map, current_depth) do + # Assign current depth to all nodes at this level + updated_depth_map = Enum.reduce(nodes, depth_map, fn {key, _data}, acc -> + Map.put(acc, key, current_depth) + end) + + # Find all children of current nodes + children = Enum.flat_map(nodes, fn {key, data} -> + data.children + |> Enum.map(fn child_key -> + child_data = Map.get(data_map, child_key) + if child_data do + {child_key, child_data} + else + nil + end + end) + |> Enum.filter(& &1) + end) + + # Recurse for children if any exist + if children != [] do + compute_depths_recursive(children, data_map, updated_depth_map, current_depth + 1) + else + updated_depth_map + end + end + defp extract_content(%{module: Scenic.Primitive.Text, data: text}), do: text defp extract_content(_), do: nil From 9ab1df43a2539f4231c1e305d01a4f67b7db2ca6 Mon Sep 17 00:00:00 2001 From: JediLuke <1879634+JediLuke@users.noreply.github.com> Date: Sat, 4 Oct 2025 22:13:48 -0500 Subject: [PATCH 09/11] add observe input --- lib/scenic/dev_tools.ex | 8 +++---- lib/scenic/scene.ex | 48 ++++++++++++++++++++++++++++++++++++++++- lib/scenic/view_port.ex | 2 +- 3 files changed, 52 insertions(+), 6 deletions(-) diff --git a/lib/scenic/dev_tools.ex b/lib/scenic/dev_tools.ex index e33595ba..6d53a896 100644 --- a/lib/scenic/dev_tools.ex +++ b/lib/scenic/dev_tools.ex @@ -1225,7 +1225,7 @@ defmodule Scenic.DevTools do end end - defp show_scene_detail(analysis, scene_filter, detailed) do + defp show_scene_detail(analysis, scene_filter, _detailed) do scene = Enum.find(analysis.scenes, fn s -> String.contains?(String.downcase(s.scene_name), String.downcase(scene_filter)) or String.contains?(s.graph_key, scene_filter) @@ -1295,7 +1295,7 @@ defmodule Scenic.DevTools do IO.puts("#{prefix}📊 #{short_id(key)}#{type_info}") # Show children - children = Enum.filter(all_entries, fn {child_key, child_data} -> + children = Enum.filter(all_entries, fn {_child_key, child_data} -> child_data.parent == key end) @@ -1303,7 +1303,7 @@ defmodule Scenic.DevTools do |> Enum.each(fn {{child_key, child_data}, idx} -> is_last = idx == length(children) - 1 child_prefix = prefix <> if is_last, do: "└── ", else: "├── " - next_prefix = prefix <> if is_last, do: " ", else: "│ " + _next_prefix = prefix <> if is_last, do: " ", else: "│ " render_hierarchy_tree(child_key, child_data, all_entries, child_prefix, verbose) end) @@ -1436,7 +1436,7 @@ defmodule Scenic.DevTools do "#{type_info}#{semantic_info}#{content_info}" end - defp suggest_available_options(opts) do + defp suggest_available_options(_opts) do # This would show what options are actually available # For now, just show a helpful message IO.puts("\n💡 Try:") diff --git a/lib/scenic/scene.ex b/lib/scenic/scene.ex index bef8135f..e27e61f1 100644 --- a/lib/scenic/scene.ex +++ b/lib/scenic/scene.ex @@ -681,6 +681,12 @@ defmodule Scenic.Scene do GenServer.cast(pid, msg) end + @doc "Call a message to a scene's parent synchronously" + @spec call_parent(scene :: Scene.t(), msg :: any) :: any + def call_parent(%Scene{parent: pid}, msg) do + GenServer.call(pid, msg) + end + @doc "Cast a message to a scene's children" @spec send_children(scene :: Scene.t(), msg :: any) :: :ok | {:error, :no_children} def send_children(%Scene{children: nil}, _msg), do: {:error, :no_children} @@ -1017,6 +1023,29 @@ defmodule Scenic.Scene do This has replaced push_graph() as the preferred way to push a graph. """ + @doc """ + Invoked to observe input before it's routed to handlers. + + This callback is called BEFORE `handle_input` and allows scenes to observe + input without consuming it. Always returns the scene unchanged - any return + value other than `{:noreply, scene}` is ignored. + + Use this for debugging, visualization, logging, or telemetry without + interfering with normal input routing. + + ## Example + + def observe_input({:cursor_button, {:btn_left, 1, [], coords}}, _id, scene) do + # Visualize clicks without consuming them + send(self(), {:visualize_click, coords}) + {:noreply, scene} + end + + def observe_input(_input, _id, scene), do: {:noreply, scene} + """ + @callback observe_input(input :: Scenic.ViewPort.Input.t(), id :: any, scene :: Scene.t()) :: + {:noreply, scene} when scene: Scene.t() + @callback handle_input(input :: Scenic.ViewPort.Input.t(), id :: any, scene :: Scene.t()) :: {:noreply, scene} | {:noreply, scene} @@ -1152,7 +1181,8 @@ defmodule Scenic.Scene do | {:noreply, scene, timeout() | :hibernate | {:continue, term()}} when scene: Scene.t() - @optional_callbacks handle_event: 3, + @optional_callbacks observe_input: 3, + handle_event: 3, handle_input: 3, handle_get: 2, handle_put: 2, @@ -1381,6 +1411,22 @@ defmodule Scenic.Scene do {:_input, input, raw_input, id}, %Scene{module: module, viewport: %{pid: vp_pid}} = scene ) do + IO.puts("🔍 DEBUG: Scene #{inspect(module)} received input: #{inspect(input)}") + + # First, call observe_input if it exists (for non-consuming observation) + scene = case Kernel.function_exported?(module, :observe_input, 3) do + true -> + IO.puts("🔍 DEBUG: #{inspect(module)} HAS observe_input, calling it...") + case module.observe_input(input, id, scene) do + {:noreply, %Scene{} = scene} -> scene + _ -> scene # Ignore any other return value + end + false -> + IO.puts("🔍 DEBUG: #{inspect(module)} does NOT have observe_input") + scene + end + + # Then, call handle_input as normal case Kernel.function_exported?(module, :handle_input, 3) do true -> case module.handle_input(input, id, scene) do diff --git a/lib/scenic/view_port.ex b/lib/scenic/view_port.ex index 943f8c21..ef678c67 100644 --- a/lib/scenic/view_port.ex +++ b/lib/scenic/view_port.ex @@ -2357,7 +2357,7 @@ defmodule Scenic.ViewPort do end) # Find all children of current nodes - children = Enum.flat_map(nodes, fn {key, data} -> + children = Enum.flat_map(nodes, fn {_key, data} -> data.children |> Enum.map(fn child_key -> child_data = Map.get(data_map, child_key) From 6d5114f96311e737b847b25054afee68c9ce9e98 Mon Sep 17 00:00:00 2001 From: JediLuke <1879634+JediLuke@users.noreply.github.com> Date: Sun, 12 Oct 2025 16:52:42 -0500 Subject: [PATCH 10/11] enhanced semantic def --- lib/scenic/component/button.ex | 30 ++++++--- lib/scenic/view_port.ex | 111 ++++++++++++++++++++++++++++++--- 2 files changed, 124 insertions(+), 17 deletions(-) diff --git a/lib/scenic/component/button.ex b/lib/scenic/component/button.ex index 1791f8f3..ad008e0c 100644 --- a/lib/scenic/component/button.ex +++ b/lib/scenic/component/button.ex @@ -208,10 +208,23 @@ defmodule Scenic.Component.Button do vpos = height / 2 + ascent / 2 + descent / 3 - # build the graph + # build the graph - add semantic info to make button clickable via MCP + semantic_opts = if id do + [ + semantic: %{ + type: :button, + label: text, + clickable: true, + bounds: %{left: 0, top: 0, width: width, height: height} + } + ] + else + [] + end + graph = Graph.build(font: font, font_size: font_size) - |> rrect({width, height, radius}, fill: theme.background, id: :btn, input: :cursor_button) + |> rrect({width, height, radius}, [fill: theme.background, id: id || :btn, input: :cursor_button] ++ semantic_opts) |> do_aligned_text(alignment, text, theme.text, width, vpos) # special case the dark and light themes to show an outline |> do_special_theme_outline(theme, theme.border) @@ -225,7 +238,6 @@ defmodule Scenic.Component.Button do theme: theme, id: id, text: text, - theme: theme, opts: opts ) |> assign_new(pressed: false) @@ -304,9 +316,9 @@ defmodule Scenic.Component.Button do @impl Scenic.Scene def handle_input( {:cursor_button, {:btn_left, 1, _, _}}, - :btn, + button_id, %Scene{assigns: %{id: id, graph: graph, theme: theme}} = scene - ) do + ) when button_id == id or button_id == :btn do :ok = capture_input(scene, :cursor_button) :ok = send_parent_event(scene, {:btn_pressed, id}) @@ -345,9 +357,9 @@ defmodule Scenic.Component.Button do # released inside the button def handle_input( {:cursor_button, {:btn_left, 0, _, _}}, - :btn, + button_id, %Scene{assigns: %{pressed: true, id: id, graph: graph, theme: theme}} = scene - ) do + ) when button_id == id or button_id == :btn do :ok = release_input(scene) :ok = send_parent_event(scene, {:click, id}) :ok = send_parent_event(scene, {:btn_released, id}) @@ -383,7 +395,9 @@ defmodule Scenic.Component.Button do end # ignore other input - def handle_input(_input, _id, scene) do + def handle_input(input, id, scene) do + require Logger + Logger.info("🔘 Button handle_input catch-all: #{inspect(input)}, id: #{inspect(id)}") {:noreply, scene} end diff --git a/lib/scenic/view_port.ex b/lib/scenic/view_port.ex index ef678c67..006f6cc0 100644 --- a/lib/scenic/view_port.ex +++ b/lib/scenic/view_port.ex @@ -689,12 +689,52 @@ defmodule Scenic.ViewPort do ViewPort.get_semantic(viewport, :specific_graph) # => {:ok, %{...}} """ - @spec get_semantic(viewport :: ViewPort.t(), graph_key :: any) :: + @spec get_semantic(viewport :: ViewPort.t(), graph_key :: any) :: {:ok, map} | {:error, :no_semantic_info} def get_semantic(%ViewPort{pid: pid}, graph_key \\ :main) do GenServer.call(pid, {:get_semantic, graph_key}) end + @doc """ + Register a semantic element in the viewport's semantic table. + + This allows components and scenes to manually register clickable elements + with their semantic information and bounds for testing and automation. + + ## Parameters + - `viewport` - The ViewPort struct or PID + - `graph_key` - The graph key (typically the scene name or :_root_) + - `element_id` - The semantic ID for this element (atom) + - `semantic_data` - Map containing element metadata including: + - `:type` - Element type (e.g., :button, :text_input) + - `:label` - Human-readable label + - `:clickable` - Boolean indicating if element accepts clicks + - `:bounds` - Map with :left, :top, :width, :height + + ## Examples + ViewPort.register_semantic(viewport, :_root_, :my_button, %{ + type: :button, + label: "Click Me", + clickable: true, + bounds: %{left: 100, top: 200, width: 150, height: 40} + }) + """ + @spec register_semantic( + viewport :: ViewPort.t() | pid(), + graph_key :: any(), + element_id :: atom(), + semantic_data :: map() + ) :: :ok + def register_semantic(viewport, graph_key, element_id, semantic_data) + + def register_semantic(%ViewPort{pid: pid}, graph_key, element_id, semantic_data) do + register_semantic(pid, graph_key, element_id, semantic_data) + end + + def register_semantic(pid, graph_key, element_id, semantic_data) when is_pid(pid) do + GenServer.call(pid, {:register_semantic, graph_key, element_id, semantic_data}) + end + # -------------------------------------------------------- @doc """ Inspect all semantic information in the viewport. @@ -1328,6 +1368,40 @@ defmodule Scenic.ViewPort do {:reply, result, state} end + # Register a semantic element + def handle_call({:register_semantic, graph_key, element_id, semantic_data}, _from, %{semantic_table: semantic_table} = state) do + # Get current semantic data for this graph, or create new + current_data = case :ets.lookup(semantic_table, graph_key) do + [{^graph_key, data}] -> data + [] -> %{ + graph_key: graph_key, + timestamp: System.system_time(:millisecond), + elements: %{}, + by_type: %{} + } + end + + # Add the new element + element_type = Map.get(semantic_data, :type, :unknown) + + updated_data = current_data + |> put_in([:elements, element_id], Map.merge(semantic_data, %{id: element_id})) + |> update_in([:by_type, element_type], fn existing -> + existing = existing || [] + if element_id in existing do + existing + else + [element_id | existing] + end + end) + |> Map.put(:timestamp, System.system_time(:millisecond)) + + # Store back in ETS + :ets.insert(semantic_table, {graph_key, updated_data}) + + {:reply, :ok, state} + end + def handle_call(invalid, from, %{name: name} = state) do Logger.error(""" ViewPort #{inspect(name || self())} ignored bad call @@ -1825,7 +1899,12 @@ defmodule Scenic.ViewPort do {:cursor_button, {button, action, mods, gxy}} = input, %{input_lists: ils} ) do - with {:ok, pid, xy, _inv_tx, id} <- input_find_hit(ils, :cursor_button, @root_id, gxy) do + require Logger + result = input_find_hit(ils, :cursor_button, @root_id, gxy) + Logger.info("📬 do_listed_input result: #{inspect(result)}") + + with {:ok, pid, xy, _inv_tx, id} <- result do + Logger.info("📤 Sending input to PID #{inspect(pid)}, id: #{inspect(id)}, coords: #{inspect(xy)}") send(pid, {:_input, {:cursor_button, {button, action, mods, xy}}, input, id}) end end @@ -2015,11 +2094,16 @@ defmodule Scenic.ViewPort do end defp input_find_hit(lists, input_type, name, global_point, parent_tx) do + # require Logger + # Logger.info("🎯 input_find_hit: name=#{inspect(name)}, type=#{inspect(input_type)}, point=#{inspect(global_point)}") + case Map.fetch(lists, name) do {:ok, {in_list, _, _}} -> + # Logger.info(" Found input_list with #{length(in_list)} items") do_find_hit(in_list, input_type, global_point, lists, name, parent_tx) _ -> + # Logger.info(" No input_list found for #{inspect(name)}") :not_found end end @@ -2036,17 +2120,22 @@ defmodule Scenic.ViewPort do name, parent_tx ) do + # require Logger + # Logger.info("🔍 Component hit test: name=#{inspect(name)}, component_id=#{inspect(data)}, point=#{inspect(global_point)}") + # calculate the local matrix, which becomes the parent of the component local_tx = Math.Matrix.mul(parent_tx, local_tx) # recurse to test the component case input_find_hit(lists, input_type, data, global_point, local_tx) do {:ok, _, _, _, _} = hit -> - # Rhere was a hit inside the component. Return result as we are done. + # There was a hit inside the component. Return result as we are done. + # Logger.info("✅ Component hit found!") hit :not_found -> # if not found, keep going + # Logger.info("❌ Component hit not found, continuing...") do_find_hit(tail, input_type, global_point, lists, name, parent_tx) end end @@ -2091,20 +2180,24 @@ defmodule Scenic.ViewPort do # Build semantic information from a graph defp build_semantic_info(graph, graph_key) do - elements = + elements = graph.primitives |> Enum.reduce(%{}, fn {id, primitive}, acc -> # Extract semantic data if present - use direct access for struct fields # Check if primitive has opts field and it contains semantic data - semantic = case Map.get(primitive, :opts) do - nil -> nil + opts = Map.get(primitive, :opts, []) + semantic = case opts do opts when is_list(opts) -> Keyword.get(opts, :semantic) _ -> nil end - + if semantic do + # The ID is stored in primitive.id field, not in opts + symbolic_id = Map.get(primitive, :id, id) + element_info = %{ - id: id, + id: symbolic_id, + primitive_id: id, type: primitive.module, semantic: semantic, # Extract text content for text primitives @@ -2112,7 +2205,7 @@ defmodule Scenic.ViewPort do # Store transform for position info if needed transforms: primitive.transforms } - Map.put(acc, id, element_info) + Map.put(acc, symbolic_id, element_info) else acc end From a2db350ef40e9d377fa6daebf59049df7e59a7af Mon Sep 17 00:00:00 2001 From: Luke Date: Sat, 29 Nov 2025 09:52:04 -0600 Subject: [PATCH 11/11] Updates to semantic registration --- guides/overview_graph.md | 21 ++ guides/overview_viewport.md | 155 ++++++++++- guides/testing_and_automation.md | 359 +++++++++++++++++++++++++ guides/welcome.md | 1 + lib/scenic/semantic/compiler.ex | 351 ++++++++++++++++++++++++ lib/scenic/view_port.ex | 78 +++++- lib/scenic/view_port/semantic.ex | 349 ++++++++++++++++++++++++ test/scenic/semantic/compiler_test.exs | 187 +++++++++++++ 8 files changed, 1498 insertions(+), 3 deletions(-) create mode 100644 guides/testing_and_automation.md create mode 100644 lib/scenic/semantic/compiler.ex create mode 100644 lib/scenic/view_port/semantic.ex create mode 100644 test/scenic/semantic/compiler_test.exs diff --git a/guides/overview_graph.md b/guides/overview_graph.md index 6f3e29b6..9e8ba4ea 100644 --- a/guides/overview_graph.md +++ b/guides/overview_graph.md @@ -87,6 +87,27 @@ This time, we've assigned ids to both of the text primitives. This makes it easy Notice that the graph is modified multiple times in the pipeline. +## IDs Enable Testing and Automation + +Beyond modification, IDs serve another critical purpose: they enable automated testing and AI-driven interaction with your application. + +When you assign an ID to a primitive, Scenic automatically registers it in a semantic element registry. This lets you find and interact with elements by ID rather than hardcoded coordinates: + +```elixir +# In your graph +@graph Graph.build() + |> rectangle({100, 40}, id: :save_button, translate: {200, 100}) + |> text("Click Me", id: :button_label, translate: {210, 115}) + +# In your tests or automation scripts +{:ok, viewport} = Scenic.ViewPort.info(:main_viewport) +{:ok, coords} = Scenic.ViewPort.Semantic.click_element(viewport, :save_button) +#=> {:ok, {250.0, 120.0}} # Automatically calculated center! +``` + +This is similar to how Playwright and Puppeteer work with web browsers - you can click elements by ID without knowing where they are on screen. Layout changes don't break your tests. + +[Read more about testing and automation here.](testing_and_automation.html) ## What to read next? diff --git a/guides/overview_viewport.md b/guides/overview_viewport.md index 6ab77537..f9abf0f6 100644 --- a/guides/overview_viewport.md +++ b/guides/overview_viewport.md @@ -1,5 +1,156 @@ # ViewPort Overview -Give an overview of a viewport here +The ViewPort is the central coordinator in Scenic that connects Scenes to Drivers. It manages the rendering pipeline, input routing, and maintains the semantic element registry for testing and automation. -Coming soon \ No newline at end of file +## Core Responsibilities + +### 1. Graph Management + +When a Scene calls `push_graph/2`, the ViewPort: +- Compiles the graph into a binary script (via GraphCompiler) +- Stores the script in an ETS table for fast access +- Sends the script to all connected drivers for rendering +- Compiles semantic metadata for testing (in parallel, zero overhead) + +### 2. Input Routing + +User input flows through the ViewPort: +- Driver captures input (mouse, keyboard, etc.) +- ViewPort receives input events +- Input is routed to appropriate Scenes based on focus and capture state +- Scenes handle input and update their graphs + +### 3. Driver Coordination + +The ViewPort manages one or more drivers: +- Drivers handle actual rendering (OpenGL, etc.) +- Multiple drivers can connect (multi-monitor support) +- ViewPort ensures all drivers receive graph updates +- Drivers report size, capabilities, and input back to ViewPort + +### 4. Semantic Element Registry + +The ViewPort maintains a semantic registry of UI elements for testing and automation: +- Elements with IDs are automatically registered +- Fast lookup by ID via ETS tables +- Hierarchical relationships tracked +- Enables Playwright-like testing + +[Read more about testing and automation here.](testing_and_automation.html) + +## Starting a ViewPort + +ViewPorts are typically started by your application supervisor: + +```elixir +children = [ + {Scenic.ViewPort, + name: :main_viewport, + size: {800, 600}, + default_scene: MyApp.Scene.Main, + drivers: [ + [module: Scenic.Driver.Local] + ]}, + # ... other children +] + +Supervisor.start_link(children, strategy: :one_for_one) +``` + +## Configuration Options + +- `:name` - Atom name for the viewport (required) +- `:size` - `{width, height}` tuple (required) +- `:default_scene` - Initial scene module to display +- `:drivers` - List of driver configurations +- `:semantic_registration` - Enable/disable semantic system (default: `true`) + +## Querying ViewPort Info + +Get the current ViewPort state: + +```elixir +{:ok, viewport} = Scenic.ViewPort.info(:main_viewport) + +# Access fields +viewport.size #=> {800, 600} +viewport.pid #=> #PID<0.123.0> +viewport.script_table #=> ETS table reference +viewport.semantic_table #=> ETS table reference (if enabled) +``` + +## Common Operations + +### Setting a Scene + +```elixir +Scenic.ViewPort.set_root(:main_viewport, MyApp.Scene.Other) +``` + +### Updating Graph + +From within a Scene: + +```elixir +def handle_info(:update, scene) do + graph = + scene.assigns.graph + |> Graph.modify(:my_text, &text(&1, "Updated!")) + + scene = push_graph(scene, graph) + {:noreply, scene} +end +``` + +### Sending Input (Testing) + +```elixir +# Send input directly (for testing) +input = {:cursor_button, {:btn_left, 1, [], {100, 50}}} +Scenic.ViewPort.input(:main_viewport, input) +``` + +For production testing, use the semantic system instead: + +```elixir +{:ok, viewport} = Scenic.ViewPort.info(:main_viewport) +{:ok, coords} = Scenic.ViewPort.Semantic.click_element(viewport, :my_button) +``` + +## Architecture + +``` +┌─────────────────────────────────────────────┐ +│ Application │ +└─────────────────┬───────────────────────────┘ + │ + ┌─────────▼─────────┐ + │ ViewPort │ + │ │ + │ - Script Table │ + │ - Semantic Table │ + │ - Input Routing │ + └─────────┬─────────┘ + │ + ┌─────────┴─────────┐ + │ │ + ┌────▼────┐ ┌──────▼──────┐ + │ Scene │ │ Driver │ + │ │ │ │ + │ Graph │──────▶│ Rendering │ + └─────────┘ └─────────────┘ +``` + +## Performance + +The ViewPort is designed for high performance: +- **ETS tables** - O(1) lookups for scripts and semantic elements +- **Parallel compilation** - Semantic compilation doesn't block rendering +- **Minimal copying** - Scripts shared via ETS, not message passing +- **Configurable** - Semantic system can be disabled if not needed + +## What to Read Next? + +- [Testing and Automation](testing_and_automation.html) - Semantic element system +- [Scene Overview](overview_scene.html) - Building Scenes +- [Driver Overview](overview_driver.html) - Understanding Drivers \ No newline at end of file diff --git a/guides/testing_and_automation.md b/guides/testing_and_automation.md new file mode 100644 index 00000000..06865775 --- /dev/null +++ b/guides/testing_and_automation.md @@ -0,0 +1,359 @@ +# Testing and Automation + +Scenic includes a semantic element registration system that enables automated testing and AI-driven interaction with your applications. Similar to how Playwright and Puppeteer work with web browsers, Scenic's semantic system lets you find and interact with UI elements by ID rather than hardcoded screen coordinates. + +## Why Semantic Registration? + +Traditional GUI testing requires knowing exact pixel coordinates to click buttons or interact with elements. This is fragile and breaks when layouts change. Semantic registration solves this by: + +- **Finding elements by ID** - Just like `document.getElementById()` in web development +- **Automatic bounds calculation** - No need to track where elements are on screen +- **Resilient to layout changes** - Element IDs stay stable even if positions change +- **AI automation ready** - Enables tools like MCP (Model Context Protocol) to control your app + +## Quick Start + +### 1. Add IDs to Your Elements + +Simply add an `:id` option to any primitive you want to interact with: + +```elixir +@graph Graph.build() + |> text("Click Me", id: :save_button, translate: {100, 50}) + |> rectangle({200, 40}, id: :input_field, translate: {100, 100}) + |> circle(25, id: :status_indicator, translate: {50, 50}) +``` + +That's it! Any primitive with an `:id` is automatically registered in the semantic system. + +### 2. Query Elements + +Use the `Scenic.ViewPort.Semantic` module to find and interact with elements: + +```elixir +# Get viewport +{:ok, viewport} = Scenic.ViewPort.info(:main_viewport) + +# Find element by ID +{:ok, button} = Scenic.ViewPort.Semantic.find_element(viewport, :save_button) + +# See what you got +IO.inspect(button) +#=> %Scenic.Semantic.Compiler.Entry{ +# id: :save_button, +# type: :text, +# label: "Click Me", +# local_bounds: %{left: 0, top: 0, width: 100, height: 20}, +# screen_bounds: %{left: 100, top: 50, width: 100, height: 20}, +# clickable: false, +# ... +# } +``` + +### 3. Click Elements by ID + +The most powerful feature - click elements without knowing their coordinates: + +```elixir +# Click the save button (calculates center point automatically) +{:ok, {x, y}} = Scenic.ViewPort.Semantic.click_element(viewport, :save_button) +#=> {:ok, {150.0, 60.0}} +``` + +This finds the element, calculates its center point, and sends mouse click events through the driver - just like a real user clicking! + +## Advanced Queries + +### Find All Clickable Elements + +```elixir +{:ok, elements} = Scenic.ViewPort.Semantic.find_clickable_elements(viewport) + +# Filter by type +{:ok, buttons} = Scenic.ViewPort.Semantic.find_clickable_elements( + viewport, + %{type: :component} +) + +# Filter by label text +{:ok, save_buttons} = Scenic.ViewPort.Semantic.find_clickable_elements( + viewport, + %{label: "Save"} +) +``` + +### Find Element at Coordinates + +```elixir +# What's at this screen position? +{:ok, element} = Scenic.ViewPort.Semantic.element_at_point(viewport, 150, 60) +``` + +### Get Hierarchical Tree + +```elixir +# Get full semantic tree starting from root +{:ok, tree} = Scenic.ViewPort.Semantic.get_semantic_tree(viewport) + +# Get subtree from specific element +{:ok, subtree} = Scenic.ViewPort.Semantic.get_semantic_tree(viewport, :my_group) +``` + +## Explicit Semantic Metadata + +For more control, add explicit semantic metadata to any primitive: + +```elixir +@graph Graph.build() + |> rectangle( + {100, 50}, + semantic: %{ + id: :custom_button, + type: :button, + clickable: true, + focusable: true, + label: "Save File", + role: :primary_action, + bounds: %{left: 0, top: 0, width: 100, height: 50} + } + ) +``` + +Available semantic fields: + +- `:id` - Element identifier (atom or string) +- `:type` - Semantic type (`:button`, `:input`, `:checkbox`, etc.) +- `:clickable` - Whether element responds to clicks (boolean) +- `:focusable` - Whether element can receive focus (boolean) +- `:label` - Human-readable label (string) +- `:role` - Semantic role (`:primary_action`, `:navigation`, etc.) +- `:bounds` - Custom bounds if auto-calculation doesn't work + +## Components and Groups + +Groups and components with IDs are also registered: + +```elixir +@graph Graph.build() + |> group( + fn g -> + g + |> rectangle({200, 100}, id: :dialog_background) + |> text("Are you sure?", id: :dialog_message) + |> rectangle({80, 30}, id: :ok_button) + |> rectangle({80, 30}, id: :cancel_button) + end, + id: :confirmation_dialog, + translate: {100, 100} + ) +``` + +Now you can query the dialog and all its children: + +```elixir +{:ok, dialog} = Scenic.ViewPort.Semantic.find_element(viewport, :confirmation_dialog) +{:ok, ok_btn} = Scenic.ViewPort.Semantic.find_element(viewport, :ok_button) + +# Dialog's parent_id is nil, ok_button's parent_id is :confirmation_dialog +``` + +## Configuration + +Semantic registration is enabled by default. To disable it: + +```elixir +# In your viewport config +ViewPort.start_link( + name: :main_viewport, + size: {800, 600}, + semantic_registration: false, # Disable semantic system + # ... other opts +) +``` + +## Performance + +The semantic system is designed for zero performance impact: + +- **Parallel compilation** - Semantic compilation runs asynchronously, never blocking rendering +- **ETS tables** - Fast in-memory lookups with read concurrency +- **Fire-and-forget** - No messages sent back to scenes +- **Lazy updates** - Only compiles when graphs change + +When disabled via config, there is literally zero overhead - the ETS tables aren't even created. + +## Testing Patterns + +### Integration Tests + +```elixir +defmodule MyApp.IntegrationTest do + use ExUnit.Case + + test "user can save document" do + # Start app + start_supervised!(MyApp.Application) + + # Get viewport + {:ok, vp} = Scenic.ViewPort.info(:main_viewport) + + # Interact with UI + {:ok, _} = Scenic.ViewPort.Semantic.click_element(vp, :new_document_button) + Process.sleep(100) + + # Type some content (you'll need to send input through driver) + # ... + + {:ok, _} = Scenic.ViewPort.Semantic.click_element(vp, :save_button) + Process.sleep(100) + + # Verify document was saved + assert File.exists?("test_doc.txt") + end +end +``` + +### AI Automation with MCP + +The semantic system integrates with Model Context Protocol (MCP) to enable AI agents to control your application: + +```typescript +// In MCP server +async function findClickableElements() { + const viewport = await getViewport(); + const elements = await Scenic.ViewPort.Semantic.find_clickable_elements(viewport); + return elements; +} + +async function clickElement(elementId: string) { + const viewport = await getViewport(); + const coords = await Scenic.ViewPort.Semantic.click_element(viewport, elementId); + return coords; +} +``` + +This enables natural language commands like: +- "Click the save button" +- "Show me all clickable elements" +- "Find the status indicator" + +## Current Limitations (Phase 1) + +The current implementation is Phase 1 of a multi-phase rollout: + +- ✅ **Automatic registration** - Elements with IDs auto-register +- ✅ **Query by ID** - Find elements quickly +- ✅ **Click by ID** - Interact without coordinates +- ✅ **Basic bounds** - Simple rectangles, circles, text +- ⚠️ **No transform calculations** - `screen_bounds` equals `local_bounds` (no translate/rotate/scale applied) +- ⚠️ **No component sub-scenes** - Components register, but not their internal graphs +- ⚠️ **Text bounds are estimates** - No font metrics yet (fixed 100x20) + +Future phases will add: +- **Phase 2**: Transform-aware coordinate calculation +- **Phase 3**: Component sub-scene handling +- **Phase 4**: Advanced features (visibility, filters, performance optimizations) + +## Entry Struct Reference + +When you query an element, you get a `Scenic.Semantic.Compiler.Entry` struct: + +```elixir +%Scenic.Semantic.Compiler.Entry{ + id: :my_button, # Element ID + type: :rect, # Primitive type + module: Scenic.Primitive.Rectangle, # Primitive module + parent_id: :my_group, # Parent element ID (or nil) + children: [], # Child element IDs + local_bounds: %{ # Bounds in local coordinates + left: 0, + top: 0, + width: 100, + height: 50 + }, + screen_bounds: %{ # Bounds in screen coordinates (Phase 1: same as local) + left: 0, + top: 0, + width: 100, + height: 50 + }, + clickable: false, # Can be clicked + focusable: false, # Can receive focus + label: nil, # Human-readable label + role: nil, # Semantic role + value: {100, 50}, # Primitive data (dimensions, text, etc.) + hidden: false, # Whether element is hidden + z_index: 0 # Depth order (higher = on top) +} +``` + +## Best Practices + +### 1. Use Descriptive IDs + +```elixir +# Good +id: :save_document_button +id: :email_input_field +id: :user_profile_avatar + +# Avoid +id: :button1 +id: :rect_a +id: :thing +``` + +### 2. Add IDs to Interactive Elements + +Focus on elements users interact with: +- Buttons +- Input fields +- Checkboxes +- List items +- Dialogs + +### 3. Use Groups for Complex UIs + +```elixir +@graph Graph.build() + |> group( + fn g -> + g + |> text("Name:", id: :name_label) + |> rectangle({200, 30}, id: :name_input) + end, + id: :name_field_group + ) +``` + +### 4. Leverage Semantic Metadata for Custom Components + +```elixir +defmodule MyApp.Component.CustomButton do + # ... + + def init(scene, {text, opts}, _) do + graph = + Graph.build() + |> rounded_rectangle( + {100, 40, 5}, + id: opts[:id], + semantic: %{ + type: :button, + clickable: true, + label: text, + role: opts[:role] || :action + } + ) + + scene = push_graph(scene, graph) + {:ok, scene} + end +end +``` + +## What to Read Next? + +- [ViewPort Overview](overview_viewport.html) - Understand the ViewPort architecture +- [Graph Overview](overview_graph.html) - Learn about scene graphs +- [Scenic MCP Documentation](../scenic_mcp/README.md) - AI automation integration diff --git a/guides/welcome.md b/guides/welcome.md index e22f9b69..4e57f4e1 100644 --- a/guides/welcome.md +++ b/guides/welcome.md @@ -27,6 +27,7 @@ If you are new to Scenic, then you should read the following guides. * [Standard Components](Scenic.Components.html) * [Styles](overview_styles.html) * [Transforms](overview_transforms.html) +* [Testing and Automation](testing_and_automation.html) * [Contributing](contributing.html) * [Code of Conduct](code_of_conduct.html) diff --git a/lib/scenic/semantic/compiler.ex b/lib/scenic/semantic/compiler.ex new file mode 100644 index 00000000..6e9482cf --- /dev/null +++ b/lib/scenic/semantic/compiler.ex @@ -0,0 +1,351 @@ +# +# Semantic Compiler for Scenic +# Compiles scene graphs into semantic trees for testing/automation +# + +defmodule Scenic.Semantic.Compiler do + @moduledoc """ + Compiles scene graphs into semantic trees with element registration. + + This module walks a Scenic graph and extracts semantic information about + elements (buttons, text fields, etc.) that have IDs or explicit semantic + metadata. This enables Playwright-like testing and AI automation. + + ## Phase 1 Limitations + + This is Phase 1 implementation with simplified coordinate calculation: + - Basic bounds from primitive data + - No transform matrix calculations yet (Phase 2) + - No component sub-scene handling yet (Phase 3) + """ + + alias Scenic.Graph + + defmodule Entry do + @moduledoc """ + Represents a semantic element in the scene graph. + + Contains all information needed to find, query, and interact with + an element programmatically. + """ + + @type bounds :: %{ + left: number(), + top: number(), + width: number(), + height: number() + } + + @type t :: %__MODULE__{ + id: atom() | binary(), + type: atom(), + module: module(), + parent_id: atom() | binary() | nil, + children: list(atom() | binary()), + local_bounds: bounds(), + screen_bounds: bounds(), + clickable: boolean(), + focusable: boolean(), + label: String.t() | nil, + role: atom() | nil, + value: any(), + hidden: boolean(), + z_index: integer() + } + + defstruct [ + :id, + :type, + :module, + :parent_id, + children: [], + local_bounds: %{left: 0, top: 0, width: 0, height: 0}, + screen_bounds: %{left: 0, top: 0, width: 0, height: 0}, + clickable: false, + focusable: false, + label: nil, + role: nil, + value: nil, + hidden: false, + z_index: 0 + ] + end + + @doc """ + Compiles a graph into semantic entries. + + ## Options + + - `:parent_id` - ID of parent scene (for component sub-scenes) + + ## Returns + + `{:ok, entries}` where entries is a list of `Entry` structs + """ + @spec compile(Graph.t(), keyword()) :: {:ok, list(Entry.t())} + def compile(%Graph{primitives: primitives}, opts \\ []) do + parent_id = opts[:parent_id] + + # Start from root primitive (uid 0) + {entries, _} = + compile_primitive( + [], + 0, + primitives, + parent_id, + 0 + ) + + {:ok, Enum.reverse(entries)} + end + + # Compile a single primitive and its children + defp compile_primitive(entries, uid, all_primitives, parent_id, z_index) do + primitive = all_primitives[uid] + + # Skip if primitive doesn't exist + if primitive == nil do + {entries, z_index} + else + # Check if this primitive should be registered + if should_register?(primitive) do + # Build semantic entry + entry = build_semantic_entry(primitive, parent_id, z_index) + + # Add to entries + entries = [entry | entries] + + # Process children with this entry as parent + compile_children( + entries, + primitive, + all_primitives, + entry.id, + z_index + 1 + ) + else + # Not registered, but process children anyway + compile_children( + entries, + primitive, + all_primitives, + parent_id, + z_index + ) + end + end + end + + # Process children primitives + defp compile_children(entries, primitive, all_primitives, parent_id, z_index) do + case primitive.module do + Scenic.Primitive.Group -> + # Group has list of child UIDs in its data + child_uids = primitive.data || [] + + Enum.reduce(child_uids, {entries, z_index}, fn child_uid, {acc_entries, acc_z} -> + compile_primitive(acc_entries, child_uid, all_primitives, parent_id, acc_z) + end) + + Scenic.Primitive.Component -> + # Components create sub-scenes - Phase 3 will handle this + # For now, just register the component itself + {entries, z_index} + + _ -> + # Other primitives don't have children + {entries, z_index} + end + end + + # Determine if a primitive should be registered in the semantic table + defp should_register?(primitive) do + # Register if: + # 1. Has explicit :id field (not :_root_) + # 2. Has explicit :semantic metadata in opts + # 3. Is a semantic primitive (button, text_field, etc.) - Phase 3 + + has_id = primitive.id != nil && primitive.id != :_root_ + has_semantic = get_in(normalize_opts(primitive.opts), [:semantic]) != nil + + has_id or has_semantic + end + + # Normalize opts to always be a map (can be a list or map) + defp normalize_opts(opts) when is_map(opts), do: opts + defp normalize_opts(opts) when is_list(opts), do: Enum.into(opts, %{}) + defp normalize_opts(_), do: %{} + + # Build a semantic entry from a primitive + defp build_semantic_entry(primitive, parent_id, z_index) do + local_bounds = calculate_local_bounds(primitive) + screen_bounds = apply_transforms(local_bounds, primitive.transforms) + + %Entry{ + id: get_semantic_id(primitive), + type: get_semantic_type(primitive), + module: primitive.module, + parent_id: parent_id, + local_bounds: local_bounds, + screen_bounds: screen_bounds, + # Extract semantic properties + clickable: is_clickable?(primitive), + focusable: is_focusable?(primitive), + label: get_label(primitive), + role: get_role(primitive), + value: primitive.data, + hidden: Map.get(primitive.styles || %{}, :hidden, false), + z_index: z_index + } + end + + # Apply transforms to local bounds to get screen bounds + # Phase 1: Only handles translate transform + defp apply_transforms(bounds, nil), do: bounds + defp apply_transforms(bounds, transforms) when map_size(transforms) == 0, do: bounds + + defp apply_transforms(bounds, transforms) do + case Map.get(transforms, :translate) do + {tx, ty} when is_number(tx) and is_number(ty) -> + %{bounds | left: bounds.left + tx, top: bounds.top + ty} + + _ -> + bounds + end + end + + # Extract semantic ID from primitive + defp get_semantic_id(primitive) do + # Scenic stores ID in primitive.id field, not in opts + primitive.id || + get_in(normalize_opts(primitive.opts), [:semantic, :id]) || + :unnamed + end + + # Determine semantic type + defp get_semantic_type(primitive) do + opts = normalize_opts(primitive.opts) + # Try explicit semantic type first + case get_in(opts, [:semantic, :type]) do + nil -> + # Infer from module + case primitive.module do + Scenic.Primitive.Component -> :component + Scenic.Primitive.Group -> :group + Scenic.Primitive.Text -> :text + Scenic.Primitive.Rectangle -> :rect + Scenic.Primitive.RoundedRectangle -> :rounded_rect + Scenic.Primitive.Circle -> :circle + Scenic.Primitive.Line -> :line + _ -> :unknown + end + + type -> + type + end + end + + # Calculate local bounds from primitive data + defp calculate_local_bounds(primitive) do + case primitive.module do + Scenic.Primitive.Rectangle -> + # Rectangle data: {width, height} + case primitive.data do + {w, h} -> %{left: 0, top: 0, width: w, height: h} + _ -> default_bounds() + end + + Scenic.Primitive.RoundedRectangle -> + # RoundedRectangle data: {width, height, radius} + case primitive.data do + {w, h, _r} -> %{left: 0, top: 0, width: w, height: h} + _ -> default_bounds() + end + + Scenic.Primitive.Circle -> + # Circle data: radius + case primitive.data do + r when is_number(r) -> + %{left: -r, top: -r, width: r * 2, height: r * 2} + + _ -> + default_bounds() + end + + Scenic.Primitive.Text -> + # Text - we don't know bounds without font metrics + # Phase 2 will improve this + %{left: 0, top: 0, width: 100, height: 20} + + Scenic.Primitive.Component -> + # Components store dimensions in opts as :width and :height + opts = normalize_opts(primitive.opts) + + # Try explicit semantic bounds first, then fall back to opts + case get_in(opts, [:semantic, :bounds]) do + %{} = bounds -> + bounds + + _ -> + # Read width/height from opts (common for buttons, etc.) + width = Map.get(opts, :width, 0) + height = Map.get(opts, :height, 0) + %{left: 0, top: 0, width: width, height: height} + end + + _ -> + default_bounds() + end + end + + defp default_bounds() do + %{left: 0, top: 0, width: 0, height: 0} + end + + # Determine if primitive is clickable + defp is_clickable?(primitive) do + opts = normalize_opts(primitive.opts) + # Explicit semantic metadata + case get_in(opts, [:semantic, :clickable]) do + nil -> + # Infer from type + case primitive.module do + Scenic.Primitive.Component -> true + _ -> false + end + + clickable -> + clickable + end + end + + # Determine if primitive is focusable + defp is_focusable?(primitive) do + get_in(normalize_opts(primitive.opts), [:semantic, :focusable]) || false + end + + # Extract label + defp get_label(primitive) do + opts = normalize_opts(primitive.opts) + # Try semantic label first + case get_in(opts, [:semantic, :label]) do + nil -> + # For text primitives, use the text itself + case primitive.module do + Scenic.Primitive.Text when is_binary(primitive.data) -> + primitive.data + + _ -> + nil + end + + label -> + label + end + end + + # Extract role + defp get_role(primitive) do + get_in(normalize_opts(primitive.opts), [:semantic, :role]) + end +end diff --git a/lib/scenic/view_port.ex b/lib/scenic/view_port.ex index df961334..e0be7ce6 100644 --- a/lib/scenic/view_port.ex +++ b/lib/scenic/view_port.ex @@ -114,12 +114,18 @@ defmodule Scenic.ViewPort do pid: pid, # name_table: reference, script_table: reference, + semantic_table: reference | nil, + semantic_index: reference | nil, + semantic_enabled: boolean(), size: {number, number} } defstruct name: nil, pid: nil, # name_table: nil, script_table: nil, + semantic_table: nil, + semantic_index: nil, + semantic_enabled: false, size: nil @viewports :scenic_viewports @@ -370,6 +376,13 @@ defmodule Scenic.ViewPort do with {:ok, script} <- GraphCompiler.compile(graph), {:ok, input_list} <- compile_input(graph) do + # Parallel semantic compilation (Phase 1) + if viewport.semantic_enabled do + Task.start(fn -> + compile_and_store_semantics(viewport, name, graph, opts) + end) + end + # write the script - but only if it has actually changed case get_script(viewport, name) do {:ok, ^script} -> @@ -503,6 +516,28 @@ defmodule Scenic.ViewPort do # name_table = :ets.new(:_vp_name_table_, [:protected]) script_table = :ets.new(:_vp_script_table_, [:public, {:read_concurrency, true}]) + # Create semantic tables if enabled (default: true) + {semantic_table, semantic_index, semantic_enabled} = + if opts[:semantic_registration] != false do + st = + :ets.new(:_vp_semantic_table_, [ + :public, + :ordered_set, + {:read_concurrency, true} + ]) + + si = + :ets.new(:_vp_semantic_index_, [ + :public, + :set, + {:read_concurrency, true} + ]) + + {st, si, true} + else + {nil, nil, false} + end + state = %{ # simple metadata about the ViewPort name: opts[:name], @@ -534,12 +569,20 @@ defmodule Scenic.ViewPort do # ets table for scripts. Public. Readable and Writable by others. The intended # use is that Scenes compile graphs in their own process and insert the scripts # in parallel to each other. (Trying to avoid serializing the VP on large messages) - # containing either script of graph data. The scripts can be read by multiple + # containing either script of graph data. The scripts can be read by multiple # drivers at the same time, so is read parallel optimized. If the public write # becomes problematic, the next step is to have the scripts compile, then send # finished scripts to the VP for writing. script_table: script_table, + # Semantic element tables for testing/automation (Phase 1) + # semantic_table: ETS table with {scene_name, element_id} -> Entry + # semantic_index: ETS table with element_id -> {scene_name, element_id} + # semantic_enabled: boolean flag + semantic_table: semantic_table, + semantic_index: semantic_index, + semantic_enabled: semantic_enabled, + # state related to input from drivers to scenes # input lists are generated when a scene pushes a graph. Primitives # that have input: true assigned to them end up in these lists which @@ -1167,10 +1210,40 @@ defmodule Scenic.ViewPort do # ============================================================================ # internal utilities + # Compile and store semantic elements (Phase 1) + defp compile_and_store_semantics(viewport, scene_name, graph, _opts) do + # Compile semantic tree + {:ok, entries} = Scenic.Semantic.Compiler.compile(graph) + + # Store in ETS tables + Enum.each(entries, fn entry -> + # Store in main semantic table (hierarchical key) + key = {scene_name, entry.id} + :ets.insert(viewport.semantic_table, {key, entry}) + + # Store in index table (flat lookup) + :ets.insert(viewport.semantic_index, {entry.id, key}) + end) + + :ok + rescue + error -> + require Logger + + Logger.warning( + "Semantic compilation failed for #{inspect(scene_name)}: #{Exception.message(error)}" + ) + + :ok + end + defp gen_info(%{ name: name, # name_table: name_table, script_table: script_table, + semantic_table: semantic_table, + semantic_index: semantic_index, + semantic_enabled: semantic_enabled, size: size }) do %ViewPort{ @@ -1178,6 +1251,9 @@ defmodule Scenic.ViewPort do name: name, # name_table: name_table, script_table: script_table, + semantic_table: semantic_table, + semantic_index: semantic_index, + semantic_enabled: semantic_enabled, size: size } end diff --git a/lib/scenic/view_port/semantic.ex b/lib/scenic/view_port/semantic.ex new file mode 100644 index 00000000..d8c96ffc --- /dev/null +++ b/lib/scenic/view_port/semantic.ex @@ -0,0 +1,349 @@ +# +# Semantic Query API for Scenic ViewPort +# Provides functions to find, query, and interact with semantic elements +# + +defmodule Scenic.ViewPort.Semantic do + @moduledoc """ + Query API for semantic elements in a Scenic ViewPort. + + This module provides functions to find and interact with elements by their + semantic IDs, similar to how Playwright/Puppeteer work with web elements. + + ## Example + + {:ok, viewport} = Scenic.ViewPort.info(:main_viewport) + + # Find element by ID + {:ok, button} = Scenic.ViewPort.Semantic.find_element(viewport, :save_button) + + # Find all clickable elements + {:ok, elements} = Scenic.ViewPort.Semantic.find_clickable_elements(viewport) + + # Click element by ID + {:ok, center} = Scenic.ViewPort.Semantic.click_element(viewport, :save_button) + """ + + alias Scenic.ViewPort + alias Scenic.Semantic.Compiler.Entry + + @doc """ + Find element by ID. + + Returns the full semantic entry with bounds and metadata. + + ## Parameters + + - `viewport` - ViewPort struct or PID + - `element_id` - Atom or string ID of the element + + ## Returns + + - `{:ok, entry}` - Entry struct with element data + - `{:error, :not_found}` - Element not found + - `{:error, :semantic_disabled}` - Semantic system not enabled + """ + @spec find_element(ViewPort.t() | pid(), atom() | binary()) :: + {:ok, Entry.t()} | {:error, atom()} + def find_element(viewport, element_id) + + def find_element(%ViewPort{} = viewport, element_id) do + if viewport.semantic_enabled do + case lookup_in_index(viewport.semantic_index, element_id) do + {:ok, key} -> + case :ets.lookup(viewport.semantic_table, key) do + [{^key, entry}] -> {:ok, entry} + [] -> {:error, :not_found} + end + + :not_found -> + {:error, :not_found} + end + else + {:error, :semantic_disabled} + end + end + + def find_element(pid, element_id) when is_pid(pid) or is_atom(pid) do + case ViewPort.info(pid) do + {:ok, viewport} -> find_element(viewport, element_id) + error -> error + end + end + + @doc """ + Find all clickable elements, optionally filtered. + + ## Parameters + + - `viewport` - ViewPort struct or PID + - `filter` - Optional map with filter criteria: + - `:id` - Match element ID (partial string match) + - `:type` - Match element type + - `:label` - Match label text (partial string match) + + ## Returns + + - `{:ok, elements}` - List of Entry structs + - `{:error, reason}` - Error occurred + """ + @spec find_clickable_elements(ViewPort.t() | pid(), map()) :: + {:ok, list(Entry.t())} | {:error, atom()} + def find_clickable_elements(viewport, filter \\ %{}) + + def find_clickable_elements(%ViewPort{} = viewport, filter) do + if viewport.semantic_enabled do + elements = + viewport.semantic_table + |> :ets.tab2list() + |> Enum.map(&elem(&1, 1)) + |> Enum.filter(& &1.clickable) + |> apply_filters(filter) + |> Enum.sort_by(& &1.z_index) + + {:ok, elements} + else + {:error, :semantic_disabled} + end + end + + def find_clickable_elements(pid, filter) when is_pid(pid) or is_atom(pid) do + case ViewPort.info(pid) do + {:ok, viewport} -> find_clickable_elements(viewport, filter) + error -> error + end + end + + @doc """ + Get element at screen coordinates. + + Returns the top-most clickable element at the given point. + + ## Parameters + + - `viewport` - ViewPort struct or PID + - `x` - X coordinate + - `y` - Y coordinate + + ## Returns + + - `{:ok, entry}` - Element at point + - `{:error, :not_found}` - No element at point + """ + @spec element_at_point(ViewPort.t() | pid(), number(), number()) :: + {:ok, Entry.t()} | {:error, atom()} + def element_at_point(viewport, x, y) + + def element_at_point(%ViewPort{} = viewport, x, y) do + if viewport.semantic_enabled do + result = + viewport.semantic_table + |> :ets.tab2list() + |> Enum.map(&elem(&1, 1)) + |> Enum.filter(fn entry -> + bounds = entry.screen_bounds + + x >= bounds.left && + x <= bounds.left + bounds.width && + y >= bounds.top && + y <= bounds.top + bounds.height + end) + |> Enum.max_by(& &1.z_index, fn -> nil end) + + case result do + nil -> {:error, :not_found} + entry -> {:ok, entry} + end + else + {:error, :semantic_disabled} + end + end + + def element_at_point(pid, x, y) when is_pid(pid) or is_atom(pid) do + case ViewPort.info(pid) do + {:ok, viewport} -> element_at_point(viewport, x, y) + error -> error + end + end + + @doc """ + Click element by ID. + + Finds the element, calculates its center, and sends mouse click events + through the driver (simulating real user input). + + ## Parameters + + - `viewport` - ViewPort struct or PID + - `element_id` - Atom or string ID of the element + + ## Returns + + - `{:ok, {x, y}}` - Clicked at coordinates + - `{:error, reason}` - Failed to click + """ + @spec click_element(ViewPort.t() | pid(), atom() | binary()) :: + {:ok, {number(), number()}} | {:error, atom()} + def click_element(viewport, element_id) + + def click_element(%ViewPort{} = viewport, element_id) do + with {:ok, element} <- find_element(viewport, element_id), + {:ok, center} <- calculate_center(element.screen_bounds), + {:ok, driver_state} <- get_driver_state(viewport) do + # Send mouse press through driver + input = {:cursor_button, {:btn_left, 1, [], center}} + Scenic.Driver.send_input(driver_state, input) + + # Small delay between press and release + Process.sleep(10) + + # Send mouse release through driver + input = {:cursor_button, {:btn_left, 0, [], center}} + Scenic.Driver.send_input(driver_state, input) + + {:ok, center} + end + end + + def click_element(pid, element_id) when is_pid(pid) or is_atom(pid) do + case ViewPort.info(pid) do + {:ok, viewport} -> click_element(viewport, element_id) + error -> error + end + end + + @doc """ + Get hierarchical tree of semantic elements. + + ## Parameters + + - `viewport` - ViewPort struct or PID + - `root_id` - Root element ID (default: "_root") + + ## Returns + + - `{:ok, tree}` - Nested map with element and children + - `{:error, reason}` - Failed to build tree + """ + @spec get_semantic_tree(ViewPort.t() | pid(), atom() | binary()) :: + {:ok, map()} | {:error, atom()} + def get_semantic_tree(viewport, root_id \\ :_root_) + + def get_semantic_tree(%ViewPort{} = viewport, root_id) do + if viewport.semantic_enabled do + tree = build_tree(viewport, root_id) + {:ok, tree} + else + {:error, :semantic_disabled} + end + end + + def get_semantic_tree(pid, root_id) when is_pid(pid) or is_atom(pid) do + case ViewPort.info(pid) do + {:ok, viewport} -> get_semantic_tree(viewport, root_id) + error -> error + end + end + + # Private helpers + + # Get driver state for sending input events + defp get_driver_state(%ViewPort{pid: viewport_pid}) do + # Get viewport state to access driver_pids + case :sys.get_state(viewport_pid, 5000) do + %{driver_pids: [driver_pid | _]} -> + # Get the first driver's state + driver_state = :sys.get_state(driver_pid, 5000) + {:ok, driver_state} + + %{driver_pids: []} -> + {:error, :no_driver} + + _ -> + {:error, :invalid_viewport_state} + end + rescue + error -> + {:error, {:driver_state_failed, Exception.message(error)}} + end + + # Lookup element key in index table + defp lookup_in_index(semantic_index, element_id) do + # Normalize ID to atom + id = + case element_id do + id when is_atom(id) -> id + id when is_binary(id) -> String.to_existing_atom(id) + end + + case :ets.lookup(semantic_index, id) do + [{^id, key}] -> {:ok, key} + [] -> :not_found + end + rescue + ArgumentError -> :not_found + end + + # Apply filter criteria to element list + defp apply_filters(elements, filter) when filter == %{} do + elements + end + + defp apply_filters(elements, filter) do + elements + |> filter_by_id(Map.get(filter, :id)) + |> filter_by_type(Map.get(filter, :type)) + |> filter_by_label(Map.get(filter, :label)) + end + + defp filter_by_id(elements, nil), do: elements + + defp filter_by_id(elements, id_filter) do + Enum.filter(elements, fn entry -> + id_str = Atom.to_string(entry.id) + String.contains?(id_str, String.trim_leading(id_filter, ":")) + end) + end + + defp filter_by_type(elements, nil), do: elements + + defp filter_by_type(elements, type) do + Enum.filter(elements, fn entry -> entry.type == type end) + end + + defp filter_by_label(elements, nil), do: elements + + defp filter_by_label(elements, label_filter) do + Enum.filter(elements, fn entry -> + entry.label != nil && String.contains?(entry.label, label_filter) + end) + end + + # Calculate center point from bounds + defp calculate_center(%{left: x, top: y, width: w, height: h}) do + {:ok, {x + w / 2, y + h / 2}} + end + + defp calculate_center(_), do: {:error, :invalid_bounds} + + # Build hierarchical tree from flat semantic table + defp build_tree(viewport, parent_id) do + case find_element(viewport, parent_id) do + {:ok, parent} -> + # Find children + children = + viewport.semantic_table + |> :ets.tab2list() + |> Enum.map(&elem(&1, 1)) + |> Enum.filter(fn entry -> entry.parent_id == parent_id end) + |> Enum.map(fn child -> build_tree(viewport, child.id) end) + + parent + |> Map.from_struct() + |> Map.put(:children, children) + + {:error, _} -> + %{id: parent_id, error: :not_found} + end + end +end diff --git a/test/scenic/semantic/compiler_test.exs b/test/scenic/semantic/compiler_test.exs new file mode 100644 index 00000000..4faa4db2 --- /dev/null +++ b/test/scenic/semantic/compiler_test.exs @@ -0,0 +1,187 @@ +defmodule Scenic.Semantic.CompilerTest do + use ExUnit.Case + alias Scenic.Graph + alias Scenic.Primitives + alias Scenic.Semantic.Compiler + alias Scenic.Semantic.Compiler.Entry + + describe "compile/2" do + test "compiles empty graph" do + graph = Graph.build() + {:ok, entries} = Compiler.compile(graph) + # Root group has no ID, so not registered + assert entries == [] + end + + test "compiles rectangle with ID" do + graph = + Graph.build() + |> Primitives.rectangle({100, 50}, id: :my_rect) + + {:ok, entries} = Compiler.compile(graph) + + assert length(entries) == 1 + entry = hd(entries) + + assert entry.id == :my_rect + assert entry.type == :rect + assert entry.module == Scenic.Primitive.Rectangle + assert entry.local_bounds == %{left: 0, top: 0, width: 100, height: 50} + end + + test "compiles circle with ID" do + graph = + Graph.build() + |> Primitives.circle(25, id: :my_circle) + + {:ok, entries} = Compiler.compile(graph) + + assert length(entries) == 1 + entry = hd(entries) + + assert entry.id == :my_circle + assert entry.type == :circle + # Circle bounds centered at origin + assert entry.local_bounds == %{left: -25, top: -25, width: 50, height: 50} + end + + test "compiles text with semantic label" do + graph = + Graph.build() + |> Primitives.text("Hello", id: :my_text) + + {:ok, entries} = Compiler.compile(graph) + + assert length(entries) == 1 + entry = hd(entries) + + assert entry.id == :my_text + assert entry.type == :text + assert entry.label == "Hello" + end + + test "ignores primitives without ID" do + graph = + Graph.build() + |> Primitives.rectangle({100, 50}) + |> Primitives.circle(25) + + {:ok, entries} = Compiler.compile(graph) + + # No IDs, so nothing should be registered + assert entries == [] + end + + test "compiles primitive with explicit semantic metadata" do + graph = + Graph.build() + |> Primitives.rectangle( + {100, 50}, + semantic: %{ + id: :custom_button, + type: :button, + clickable: true, + label: "Click Me" + } + ) + + {:ok, entries} = Compiler.compile(graph) + + assert length(entries) == 1 + entry = hd(entries) + + assert entry.id == :custom_button + assert entry.type == :button + assert entry.clickable == true + assert entry.label == "Click Me" + end + + test "compiles group with multiple children" do + graph = + Graph.build() + |> Primitives.group( + fn g -> + g + |> Primitives.rectangle({100, 50}, id: :rect1) + |> Primitives.rectangle({200, 100}, id: :rect2) + end, + id: :my_group + ) + + {:ok, entries} = Compiler.compile(graph) + + # Should have 3 entries: group + 2 rectangles + assert length(entries) == 3 + + # Find the group entry + group_entry = Enum.find(entries, fn e -> e.id == :my_group end) + assert group_entry != nil + assert group_entry.type == :group + + # Find the rectangle entries + rect1 = Enum.find(entries, fn e -> e.id == :rect1 end) + assert rect1 != nil + assert rect1.parent_id == :my_group + + rect2 = Enum.find(entries, fn e -> e.id == :rect2 end) + assert rect2 != nil + assert rect2.parent_id == :my_group + end + + test "handles hidden primitives" do + graph = + Graph.build() + |> Primitives.rectangle( + {100, 50}, + id: :hidden_rect, + hidden: true + ) + + {:ok, entries} = Compiler.compile(graph) + + assert length(entries) == 1 + entry = hd(entries) + + assert entry.id == :hidden_rect + assert entry.hidden == true + end + + test "sets z_index based on tree depth" do + graph = + Graph.build() + |> Primitives.rectangle({100, 50}, id: :rect1) + |> Primitives.group( + fn g -> + g + |> Primitives.rectangle({200, 100}, id: :rect2) + end, + id: :group1 + ) + + {:ok, entries} = Compiler.compile(graph) + + rect1 = Enum.find(entries, fn e -> e.id == :rect1 end) + group1 = Enum.find(entries, fn e -> e.id == :group1 end) + rect2 = Enum.find(entries, fn e -> e.id == :rect2 end) + + # rect1 is at root level (z=0), group1 at z=1, rect2 at z=2 + assert rect1.z_index == 0 + assert group1.z_index == 1 + assert rect2.z_index == 2 + end + end + + describe "Entry struct" do + test "has sensible defaults" do + entry = %Entry{id: :test} + + assert entry.id == :test + assert entry.clickable == false + assert entry.focusable == false + assert entry.hidden == false + assert entry.z_index == 0 + assert entry.children == [] + assert entry.local_bounds == %{left: 0, top: 0, width: 0, height: 0} + end + end +end