Merge pull request #474 from redis/sentinel

Sentinel support

Author: djanowski

.gitignore

@@ -7,6 +7,7 @@ Gemfile.lock
 /.yardoc
 /coverage/*
 /doc/
+/examples/sentinel/sentinel.conf
 /nohup.out
 /pkg/*
 /rdsrv

README.md

@@ -85,6 +85,34 @@ available on [rdoc.info][rdoc].
 
 [rdoc]: http://rdoc.info/github/redis/redis-rb/
 
+## Sentinel support
+
+The client is able to perform automatic failovers by using [Redis
+Sentinel](http://redis.io/topics/sentinel).  Make sure to run Redis 2.8+
+if you want to use this feature.
+
+To connect using Sentinel, use:
+
+```ruby
+SENTINELS = [{:host => "127.0.0.1", :port => 26380},
+             {:host => "127.0.0.1", :port => 26381}]
+
+redis = Redis.new(:url => "redis://mymaster", :sentinels => SENTINELS, :role => :master)
+```
+
+* The master name identifies a group of Redis instances composed of a master
+and one or more slaves (`mymaster` in the example).
+
+* It is possible to optionally provide a role. The allowed roles are `master`
+and `slave`. When the role is `slave`, the client will try to connect to a
+random slave of the specified master.
+
+* When using the Sentinel support you need to specify a list of sentinels to
+connect to. The list does not need to enumerate all your Sentinel instances,
+but a few so that if one is down the client will try the next one. The client
+is able to remember the last Sentinel that was able to reply correctly and will
+use it for the next requests.
+
 ## Storing objects
 
 Redis only stores strings as values. If you want to store an object, you
@@ -166,7 +194,7 @@ end
       to redis, AND
     - your own code prevents the parent process from using the redis
       connection while a child is alive
-   
+
    Improper use of `inherit_socket` will result in corrupted and/or incorrect
    responses.
 

examples/consistency.rb

@@ -0,0 +1,114 @@
+# This file implements a simple consistency test for Redis-rb (or any other
+# Redis environment if you pass a different client object) where a client
+# writes to the database using INCR in order to increment keys, but actively
+# remember the value the key should have. Before every write a read is performed
+# to check if the value in the database matches the value expected.
+#
+# In this way this program can check for lost writes, or acknowledged writes
+# that were executed.
+#
+# Copyright (C) 2013-2014 Salvatore Sanfilippo <[email protected]>
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+# 
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'redis'
+
+class ConsistencyTester
+    def initialize(redis)
+        @r = redis
+        @working_set = 10000
+        @keyspace = 100000
+        @writes = 0
+        @reads = 0
+        @failed_writes = 0
+        @failed_reads = 0
+        @lost_writes = 0
+        @not_ack_writes = 0
+        @delay = 0
+        @cached = {} # We take our view of data stored in the DB.
+        @prefix = [Process.pid.to_s,Time.now.usec,@r.object_id,""].join("|")
+        @errtime = {}
+    end
+
+    def genkey
+        # Write more often to a small subset of keys
+        ks = rand() > 0.5 ? @keyspace : @working_set
+        @prefix+"key_"+rand(ks).to_s
+    end
+
+    def check_consistency(key,value)
+        expected = @cached[key]
+        return if !expected  # We lack info about previous state.
+        if expected > value
+            @lost_writes += expected-value
+        elsif expected < value
+            @not_ack_writes += value-expected
+        end
+    end
+
+    def puterr(msg)
+        if !@errtime[msg] || Time.now.to_i != @errtime[msg]
+            puts msg
+        end
+        @errtime[msg] = Time.now.to_i
+    end
+
+    def test
+        last_report = Time.now.to_i
+        while true
+            # Read
+            key = genkey
+            begin
+                val = @r.get(key)
+                check_consistency(key,val.to_i)
+                @reads += 1
+            rescue => e
+                puterr "Reading: #{e.class}: #{e.message} (#{e.backtrace.first})"
+                @failed_reads += 1
+            end
+
+            # Write
+            begin
+                @cached[key] = @r.incr(key).to_i
+                @writes += 1
+            rescue => e
+                puterr "Writing: #{e.class}: #{e.message} (#{e.backtrace.first})"
+                @failed_writes += 1
+            end
+
+            # Report
+            sleep @delay
+            if Time.now.to_i != last_report
+                report = "#{@reads} R (#{@failed_reads} err) | " +
+                         "#{@writes} W (#{@failed_writes} err) | "
+                report += "#{@lost_writes} lost | " if @lost_writes > 0
+                report += "#{@not_ack_writes} noack | " if @not_ack_writes > 0
+                last_report = Time.now.to_i
+                puts report
+            end
+        end
+    end
+end
+
+Sentinels = [{:host => "127.0.0.1", :port => 26379},
+             {:host => "127.0.0.1", :port => 26380}]
+r = Redis.new(:url => "redis://master1", :sentinels => Sentinels, :role => :master)
+tester = ConsistencyTester.new(r)
+tester.test

examples/sentinel.rb

@@ -0,0 +1,41 @@
+require 'redis'
+
+# This example creates a master-slave setup with a sentinel, then connects to
+# it and sends write commands in a loop.
+#
+# After 30 seconds, the master dies. You will be able to see how a new master
+# is elected and things continue to work as if nothing happened.
+#
+# To run this example:
+#
+#   $ ruby -I./lib examples/sentinel.rb
+#
+
+at_exit do
+  begin
+    Process.kill(:INT, $redises)
+  rescue Errno::ESRCH
+  end
+
+  Process.waitall
+end
+
+$redises = spawn("examples/sentinel/start")
+
+Sentinels = [{:host => "127.0.0.1", :port => 26379},
+             {:host => "127.0.0.1", :port => 26380}]
+r = Redis.new(:url => "redis://master1", :sentinels => Sentinels, :role => :master)
+
+# Set keys into a loop.
+#
+# The example traps errors so that you can actually try to failover while
+# running the script to see redis-rb reconfiguring.
+(0..1000000).each{|i|
+    begin
+        r.set(i,i)
+        $stdout.write("SET (#{i} times)\n") if i % 100 == 0
+    rescue => e
+        $stdout.write("E")
+    end
+    sleep(0.01)
+}

examples/sentinel/sentinel.conf

@@ -0,0 +1,9 @@
+sentinel monitor master1 127.0.0.1 6380 2
+sentinel down-after-milliseconds master1 5000
+sentinel failover-timeout master1 15000
+sentinel parallel-syncs master1 1
+
+sentinel monitor master2 127.0.0.1 6381 2
+sentinel down-after-milliseconds master2 5000
+sentinel failover-timeout master2 15000
+sentinel parallel-syncs master2 1

examples/sentinel/start

@@ -0,0 +1,49 @@
+#! /usr/bin/env ruby
+
+# This is a helper script used together with examples/sentinel.rb
+# It runs two Redis masters, two slaves for each of them, and two sentinels.
+# After 30 seconds, the first master dies.
+#
+# You don't need to run this script yourself. Rather, use examples/sentinel.rb.
+
+require "fileutils"
+
+$pids = []
+
+at_exit do
+  $pids.each do |pid|
+    begin
+      Process.kill(:INT, pid)
+    rescue Errno::ESRCH
+    end
+  end
+
+  Process.waitall
+end
+
+base = File.expand_path(File.dirname(__FILE__))
+
+# Masters
+$pids << spawn("redis-server --port 6380 --loglevel warning")
+$pids << spawn("redis-server --port 6381 --loglevel warning")
+
+# Slaves of Master 1
+$pids << spawn("redis-server --port 63800 --slaveof 127.0.0.1 6380 --loglevel warning")
+$pids << spawn("redis-server --port 63801 --slaveof 127.0.0.1 6380 --loglevel warning")
+
+# Slaves of Master 2
+$pids << spawn("redis-server --port 63810 --slaveof 127.0.0.1 6381 --loglevel warning")
+$pids << spawn("redis-server --port 63811 --slaveof 127.0.0.1 6381 --loglevel warning")
+
+FileUtils.cp(File.join(base, "sentinel.conf"), "tmp/sentinel1.conf")
+FileUtils.cp(File.join(base, "sentinel.conf"), "tmp/sentinel2.conf")
+
+# Sentinels
+$pids << spawn("redis-server tmp/sentinel1.conf --sentinel --port 26379")
+$pids << spawn("redis-server tmp/sentinel2.conf --sentinel --port 26380")
+
+sleep 30
+
+Process.kill(:KILL, $pids[0])
+
+Process.waitall