add storage interface with Redis cluster and Memcached adapters for distributed deployments

Author: joshuay03

.github/workflows/main.yml

@@ -26,13 +26,25 @@ jobs:
     env:
       RAILS_VERSION: ${{ matrix.rails }}
       RAILS_ENV: development
+      REDIS_URL: redis://localhost:6379
+      REDIS_CLUSTER_NODE_1: redis://localhost:8000
+      REDIS_CLUSTER_NODE_2: redis://localhost:8001
+      REDIS_CLUSTER_NODE_3: redis://localhost:8002
+      MEMCACHED_URL: localhost:11211
 
     steps:
       - uses: actions/checkout@v4
+      - name: Start storage services
+        run: |
+          docker compose -f docker-compose.storage.yml up -d
+          docker compose -f docker-compose.storage.yml wait redis-cluster-init
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{ matrix.ruby }}
           bundler-cache: true
       - name: Run the default task
         run: bundle exec rake
+      - name: Stop storage services
+        if: always()
+        run: docker compose -f docker-compose.storage.yml down

.gitignore

@@ -1,4 +1,5 @@
-/.bundle/
-/tmp/
+.bundle/
+log/
+tmp/
 Gemfile.lock
 *.gem

CHANGELOG.md

@@ -1,12 +1,14 @@
 ## [Unreleased]
 
+- Add storage interface with Redis cluster and Memcached adapters for distributed deployments
+
 ## [0.4.0] - 2025-08-30
 
-- Add sampling configuration to control percentage of requests profiled
-- Replace response body buffering with streaming response wrapper
-- Make prosopite logging thread-safe with thread-local storage
-- Fix missing namespace for VERNIER_PROFILE_OUT_FILE_EXTENSION in engine routes
 - Improve error handling and input validation
+- Fix missing namespace for VERNIER_PROFILE_OUT_FILE_EXTENSION in engine routes
+- Make prosopite logging thread-safe with thread-local storage
+- Replace response body buffering with streaming response wrapper
+- Add sampling configuration to control percentage of requests profiled
 
 ## [0.3.2] - 2025-05-14
 

Gemfile

@@ -31,3 +31,7 @@ end
 gem "rails", ENV["RAILS_VERSION"]
 
 gem "rails-dom-testing"
+
+gem "redis", ">= 5.0"
+gem "redis-clustering", ">= 5.0"
+gem "dalli", ">= 3.0"

README.md

@@ -19,9 +19,7 @@ Utilizes [vernier](https://github.com/jhawthorn/vernier) for profiling and
 1. Add the gem to your Rails application's Gemfile:
 
 ```ruby
-group :development do
-  gem "dial"
-end
+gem "dial"
 ```
 
 2. Install the gem:
@@ -34,7 +32,7 @@ bundle install
 
 ```ruby
 # this will mount the engine at /dial
-mount Dial::Engine, at: "/" if Rails.env.development?
+mount Dial::Engine, at: "/"
 ```
 
 4. (Optional) Configure the gem in an initializer:
@@ -44,6 +42,8 @@ mount Dial::Engine, at: "/" if Rails.env.development?
 
 Dial.configure do |config|
   config.sampling_percentage = 50
+  config.storage = Dial::Storage::RedisAdapter
+  config.storage_options = { client: Redis.new(url: ENV["REDIS_URL"]), ttl: 86400 }
   config.vernier_interval = 100
   config.vernier_allocation_interval = 10_000
   config.prosopite_ignore_queries += [/pg_sleep/i]
@@ -55,11 +55,62 @@ end
 Option | Description | Default
 :- | :- | :-
 `sampling_percentage` | Percentage of requests to profile. | `100` in development, `1` in production
+`storage` | Storage adapter class for profile data | `Dial::Storage::FileAdapter`
+`storage_options` | Options hash passed to storage adapter | `{ ttl: 3600 }`
 `content_security_policy_nonce` | Sets the content security policy nonce to use when inserting Dial's script. Can be a string, or a Proc which receives `env` and response `headers` as arguments and returns the nonce string. | Rails generated nonce or `nil`
 `vernier_interval` | Sets the `interval` option for vernier. | `200`
 `vernier_allocation_interval` | Sets the `allocation_interval` option for vernier. | `2_000`
 `prosopite_ignore_queries` | Sets the `ignore_queries` option for prosopite. | `[/schema_migrations/i]`
 
+## Storage Backends
+
+### File Storage (Default)
+
+Profile data is stored as files on disk with polled expiration. Only suitable for development and single-server deployments.
+
+```ruby
+Dial.configure do |config|
+  config.storage = Dial::Storage::FileAdapter
+  config.storage_options = { ttl: 86400 }
+end
+```
+
+### Redis Storage
+
+Profile data is stored in Redis with automatic expiration. Supports both single Redis instances and Redis Cluster.
+
+```ruby
+# Single Redis instance
+Dial.configure do |config|
+  config.storage = Dial::Storage::RedisAdapter
+  config.storage_options = { client: Redis.new(url: "redis://localhost:6379"), ttl: 86400 }
+end
+
+# Redis Cluster
+Dial.configure do |config|
+  config.storage = Dial::Storage::RedisAdapter
+  config.storage_options = {
+    client: Redis::Cluster.new(nodes: [
+      "redis://node1:7000",
+      "redis://node2:7001",
+      "redis://node3:7002"
+    ]),
+    ttl: 86400
+  }
+end
+```
+
+### Memcached Storage
+
+Profile data is stored in Memcached with automatic expiration.
+
+```ruby
+Dial.configure do |config|
+  config.storage = Dial::Storage::MemcachedAdapter
+  config.storage_options = { client: Dalli::Client.new("localhost:11211"), ttl: 86400 }
+end
+```
+
 ## Comparison with [rack-mini-profiler](https://github.com/MiniProfiler/rack-mini-profiler)
 
 |                           | rack-mini-profiler                 | Dial                                                    |
@@ -72,7 +123,8 @@ Option | Description | Default
 | Memory Profiling          | Yes (with memory_profiler)         | Yes (*overall usage only) (via vernier hook - graph)    |
 | View Profiling            | Yes                                | Yes (via vernier hook - marker table, chart)            |
 | Snapshot Sampling         | Yes                                | No                                                      |
-| Production Support        | Yes                                | No                                                      |
+| Storage Backends          | Redis, Memcached, File, Memory     | Redis, Memcached, File                                  |
+| Production Ready          | Yes                                | Yes                                                     |
 
 > [!NOTE]
 > SQL queries displayed in the profile are not annotated with the caller location by default. If you're not using the
@@ -84,6 +136,10 @@ Option | Description | Default
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `bundle exec rake test` to run the
 tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
 
+### Testing Storage Adapters
+
+To test the Redis and Memcached storage adapters, you'll need running instances: `docker compose -f docker-compose.storage.yml up`
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/joshuay03/dial.

docker-compose.storage.yml

@@ -0,0 +1,95 @@
+x-redis-cluster: &redis-cluster-template
+  image: redis:latest
+  network_mode: host
+  command: redis-server --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 5000 --appendonly yes
+
+services:
+  redis:
+    image: redis:latest
+    ports:
+      - "6379:6379"
+    command: redis-server --appendonly yes
+    volumes:
+      - redis_data:/data
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  redis-cluster-1:
+    <<: *redis-cluster-template
+    command: redis-server --port 8000 --bind 127.0.0.1 --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 5000 --appendonly yes
+    volumes:
+      - redis_cluster_1:/data
+
+  redis-cluster-2:
+    <<: *redis-cluster-template
+    command: redis-server --port 8001 --bind 127.0.0.1 --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 5000 --appendonly yes
+    volumes:
+      - redis_cluster_2:/data
+
+  redis-cluster-3:
+    <<: *redis-cluster-template
+    command: redis-server --port 8002 --bind 127.0.0.1 --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 5000 --appendonly yes
+    volumes:
+      - redis_cluster_3:/data
+
+  redis-cluster-4:
+    <<: *redis-cluster-template
+    command: redis-server --port 8003 --bind 127.0.0.1 --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 5000 --appendonly yes
+    volumes:
+      - redis_cluster_4:/data
+
+  redis-cluster-5:
+    <<: *redis-cluster-template
+    command: redis-server --port 8004 --bind 127.0.0.1 --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 5000 --appendonly yes
+    volumes:
+      - redis_cluster_5:/data
+
+  redis-cluster-6:
+    <<: *redis-cluster-template
+    command: redis-server --port 8005 --bind 127.0.0.1 --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 5000 --appendonly yes
+    volumes:
+      - redis_cluster_6:/data
+
+  redis-cluster-init:
+    image: redis:latest
+    network_mode: host
+    depends_on:
+      - redis-cluster-1
+      - redis-cluster-2
+      - redis-cluster-3
+      - redis-cluster-4
+      - redis-cluster-5
+      - redis-cluster-6
+    command: |
+      sh -c '
+        echo "Waiting for Redis cluster nodes to be ready..."
+        until redis-cli -h 127.0.0.1 -p 8000 ping && redis-cli -h 127.0.0.1 -p 8001 ping && redis-cli -h 127.0.0.1 -p 8002 ping && redis-cli -h 127.0.0.1 -p 8003 ping && redis-cli -h 127.0.0.1 -p 8004 ping && redis-cli -h 127.0.0.1 -p 8005 ping; do
+          echo "Waiting for all Redis nodes..."
+          sleep 2
+        done
+        echo "All Redis nodes are ready. Initializing cluster..."
+        redis-cli --cluster create 127.0.0.1:8000 127.0.0.1:8001 127.0.0.1:8002 127.0.0.1:8003 127.0.0.1:8004 127.0.0.1:8005 --cluster-replicas 1 --cluster-yes
+        echo "Redis cluster initialized successfully"
+      '
+    restart: "no"
+    volumes:
+      - redis_cluster_init:/data
+
+  memcached:
+    image: memcached:latest
+    ports:
+      - "11211:11211"
+    command: memcached -m 64
+
+volumes:
+  redis_data:
+  redis_cluster_1:
+  redis_cluster_2:
+  redis_cluster_3:
+  redis_cluster_4:
+  redis_cluster_5:
+  redis_cluster_6:
+  redis_cluster_init:

lib/dial.rb

@@ -4,6 +4,7 @@
 require_relative "dial/util"
 
 require_relative "dial/configuration"
+require_relative "dial/storage"
 
 require_relative "dial/railtie"
 require_relative "dial/engine"

lib/dial/configuration.rb

@@ -12,8 +12,10 @@ def self._configuration
   class Configuration
     def initialize
       @options = {
-        sampling_percentage: ::Rails.env.development? ? SAMPLING_PERCENTAGE_DEV : SAMPLING_PERCENTAGE_PROD,
-        content_security_policy_nonce: -> (env, _headers) { env[NONCE] || EMPTY_NONCE },
+        sampling_percentage: default_sampling_percentage,
+        storage: default_storage,
+        storage_options: { ttl: STORAGE_TTL },
+        content_security_policy_nonce: -> env, _headers { env[NONCE] || EMPTY_NONCE },
         vernier_interval: VERNIER_INTERVAL,
         vernier_allocation_interval: VERNIER_ALLOCATION_INTERVAL,
         prosopite_ignore_queries: PROSOPITE_IGNORE_QUERIES,
@@ -35,5 +37,15 @@ def freeze
 
       super
     end
+
+    private
+
+    def default_sampling_percentage
+      ::Rails.env.development? ? SAMPLING_PERCENTAGE_DEV : SAMPLING_PERCENTAGE_PROD
+    end
+
+    def default_storage
+      Storage::FileAdapter
+    end
   end
 end

lib/dial/constants.rb

@@ -16,10 +16,9 @@ module Dial
   NONCE = ::ActionDispatch::ContentSecurityPolicy::Request::NONCE
   REQUEST_TIMING = "dial_request_timing"
 
-  FILE_STALE_SECONDS = 60 * 60
-
   SAMPLING_PERCENTAGE_DEV = 100
   SAMPLING_PERCENTAGE_PROD = 1
+  STORAGE_TTL = 60 * 60
   EMPTY_NONCE = ""
 
   VERNIER_INTERVAL = 200

lib/dial/engine/routes.rb

@@ -1,41 +1,41 @@
 # frozen_string_literal: true
 
+require "uri"
+
 Dial::Engine.routes.draw do
   scope path: "/dial", as: "dial" do
     get "profile", to: lambda { |env|
-      uuid = env[::Rack::QUERY_STRING].sub "uuid=", ""
-
-      # Validate UUID format (should end with _vernier)
-      unless uuid.match?(/\A[0-9a-f-]+_vernier\z/)
+      query_params = URI.decode_www_form(env[::Rack::QUERY_STRING]).to_h
+      profile_key = query_params["key"]
+      unless profile_key && profile_key.match?(/\A[0-9a-f-]+_vernier\z/i)
         return [
           400,
           { "Content-Type" => "text/plain" },
           ["Bad Request"]
         ]
       end
 
-      path = String ::Rails.root.join Dial::VERNIER_PROFILE_OUT_RELATIVE_DIRNAME, (uuid + Dial::VERNIER_PROFILE_OUT_FILE_EXTENSION)
-
-      if File.exist? path
-        begin
-          content = File.read path
+      profile_storage_key = Dial::Storage.profile_storage_key profile_key
+      begin
+        content = Dial::Storage.fetch profile_storage_key
+        if content
           [
             200,
             { "Content-Type" => "application/json", "Access-Control-Allow-Origin" => Dial::VERNIER_VIEWER_URL },
             [content]
           ]
-        rescue
+        else
           [
-            500,
+            404,
             { "Content-Type" => "text/plain" },
-            ["Internal Server Error"]
+            ["Not Found"]
           ]
         end
-      else
+      rescue
         [
-          404,
+          500,
           { "Content-Type" => "text/plain" },
-          ["Not Found"]
+          ["Internal Server Error"]
         ]
       end
     }