multiple sentinels

Author: kornrunner

agent/agent.go

@@ -57,7 +57,7 @@ func SaveTwemproxyConfig() {
 
 func RestartTwemproxy() error {
 	Debug("Restarting Twemproxy.")
-	args := strings.Split(Setting.RestartArgs, string(' '))
+	args := strings.Split(Settings.RestartArgs, string(' '))
 	out, err := exec.Command(Settings.RestartCommand, args...).Output()
 
 	if err != nil {
@@ -67,9 +67,19 @@ func RestartTwemproxy() error {
 	return err
 }
 
-func GetSentinel() (sentinel string) {
-	address := ComposeRedisAddress(Settings.SentinelIp, Settings.SentinelPort)
-	return address
+func GetSentinel() (redis.Conn, error) {
+	var c redis.Conn
+	var err error
+	for i := range Settings.Sentinels {
+		c, err = redis.Dial("tcp", Settings.Sentinels[i])
+		if err == nil {
+			Debug(fmt.Sprintf("Connected to sentinel %s", Settings.Sentinels[i]))
+			break;
+		}
+		Debug(fmt.Sprintf("Sentinel %s is not reachable", Settings.Sentinels[i]))
+	}
+
+	return c, err
 }
 
 func SwitchMaster(master_name string, ip string, port string) error {
@@ -84,7 +94,7 @@ func SwitchMaster(master_name string, ip string, port string) error {
 }
 
 func ValidateCurrentMaster() error {
-	c, err := redis.Dial("tcp", GetSentinel())
+	c, err := GetSentinel()
 	if err != nil {
 		return err
 	}
@@ -111,10 +121,9 @@ func ValidateCurrentMaster() error {
 }
 
 func SubscribeToSentinel() {
-	sentinel := GetSentinel()
-	c, err := redis.Dial("tcp", sentinel)
+	c, err := GetSentinel()
 	if err != nil {
-		Fatal("Cannot connect to redis sentinel:", sentinel)
+		Fatal("Cannot connect to any sentinel.")
 	}
 
 	err = ValidateCurrentMaster()
@@ -133,7 +142,15 @@ func SubscribeToSentinel() {
 		case redis.Subscription:
 			Debug(fmt.Sprintf("%s: %s %d", v.Channel, v.Kind, v.Count))
 		case error:
-			Fatal("Error with redis connection:", psc)
+			Debug("Subscription error, trying to fallback")
+			c, err = GetSentinel()
+			if err != nil {
+				Fatal("Error with redis connection:", psc)
+			}
+
+			psc = redis.PubSubConn{c}
+			Debug("Subscribing to sentinel (+switch-master).")
+			psc.Subscribe("+switch-master")
 		}
 	}
 }

agent/settings.go

@@ -10,8 +10,7 @@ type AgentSettings struct {
 	Verbose             bool
 	TwemproxyPoolName   string `yaml:"twemproxy_pool_name"`
 	TwemproxyConfigFile string `yaml:"twemproxy_config_file"`
-	SentinelIp          string `yaml:"sentinel_ip"`
-	SentinelPort        string `yaml:"sentinel_port"`
+	Sentinels           []string `yaml:"sentinels"`
 	RestartCommand      string `yaml:"restart_command"`
 	RestartArgs         string `yaml:"restart_args"`
 	LogFile             string `yaml:"log_file"`
@@ -22,8 +21,7 @@ var Settings AgentSettings = AgentSettings{}
 func ValidateSettings() {
 	if Settings.TwemproxyPoolName == "" ||
 		Settings.TwemproxyConfigFile == "" ||
-		Settings.SentinelIp == "" ||
-		Settings.SentinelPort == "" ||
+		len(Settings.Sentinels) == 0 ||
 		Settings.RestartCommand == "" {
 		flag.Usage()
 		os.Exit(1)

conf/agent.yml

@@ -1,7 +1,7 @@
 twemproxy_pool_name: "root"
 twemproxy_config_file: "conf/nutcracker.yml"
-sentinel_ip: "127.0.0.1"
-sentinel_port: "26379"
+sentinels:
+  - "127.0.0.1:26379"
 restart_command: "/etc/init.d/twemproxy"
 restart_args: "restart"
 log_file: "agent.log"