Improve and actualize orchestrator-agent , extract many LVM commands into configuration file

README.md

@@ -96,6 +96,13 @@ The following is a complete list of configuration parameters:
 * `SnapshotMountPoint`                 (string), a known mountpoint onto which a `mount` command will mount snapshot volumes
 * `ContinuousPollSeconds`              (uint), internal clocking interval (default 60 seconds)
 * `ResubmitAgentIntervalMinutes`       (uint), interval at which the agent re-submits itself to *orchestrator* daemon
+* `LogicalVolumesCommand`              (string), command which list logical volumes, default implementation used lvs
+* `GetMountCommand`                    (string), command which get mount point parameters by mount point name, default implementation over cat /etc/mtab
+* `UnmountCommand`                     (string), command which unmount current mount point, default implementation just execute umount
+* `MountLVCommand`                     (string), command which mount selected snapshot, default implementation execute mount selected LVM snapshot 
+* `RemoveLVCommand`                    (string), command which remove selected snapshot, default implementation execute lvremove selected LVM snapshot
+* `MySQLTailErrorLogCommand`           (string), command which return last 20 lines from MySQL @@log_error file
+* `GetLogicalVolumeFSTypeCommand`      (string), command which return logical volume filesystem type
 * `CreateSnapshotCommand`              (string), command which creates new LVM snapshot of MySQL data
 * `AvailableLocalSnapshotHostsCommand` (string), command which returns list of hosts in local DC on which recent snapshots are available
 * `AvailableSnapshotHostsCommand`      (string), command which returns list of hosts in all DCs on which recent snapshots are available
@@ -111,6 +118,7 @@ The following is a complete list of configuration parameters:
 * `PostCopyCommand`                    (string), command to be executed after the seed is complete (cleanup)
 * `AgentsServer`                       (string), **Required** URL of your **orchestrator** daemon, You must add the port the orchestrator server expects to talk to agents to (see below, e.g. `https://my.orchestrator.daemon:3001`)
 * `HTTPPort`                           (uint),   Port to listen on  
+* `SeedTransferPort`                   (uint),   TCP Port to seed data transfer  
 * `HTTPAuthUser`                       (string), Basic auth user (default empty, meaning no auth)
 * `HTTPAuthPassword`                   (string), Basic auth password
 * `UseSSL`                             (bool),   If `true` then serving via `https` protocol
@@ -127,6 +135,8 @@ An example configuration file may be:
     "AgentsServer": "https://my.orchestrator.daemon:3001",
     "ContinuousPollSeconds" : 60,
     "ResubmitAgentIntervalMinutes": 60,
+    "LogicalVolumesCommand": "lvs --noheading -o lv_name,vg_name,lv_path,snap_percent",
+    "GetMountCommand":       "grep %s /etc/mtab", 
     "CreateSnapshotCommand":                "/path/to/snapshot-command.bash",
     "AvailableLocalSnapshotHostsCommand":   "/path/to/snapshot-local-availability-command.bash",
     "AvailableSnapshotHostsCommand":        "/path/to/snapshot-availability-command.bash",

Vagrantfile

@@ -15,7 +15,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     config.vm.box = BOX
     config.vm.box_download_insecure = true
     config.vm.box_check_update = false
-    config.vm.synced_folder '.', '/orchestrator-agent', type: 'rsync', rsync__auto: true
+    config.vm.synced_folder '.', '/orchestrator-agent'
+    #, type: 'rsync', rsync__auto: true
 
     config.vm.hostname = "orchestrator-agent"
     config.vm.network "private_network", ip: "192.168.57.211", virtualbox__inet: true

conf/orchestrator-agent.conf.json

@@ -4,6 +4,8 @@
     "AgentsServerPort": ":3001",
     "ContinuousPollSeconds" : 60,
     "ResubmitAgentIntervalMinutes": 60,
+    "LogicalVolumesCommand": "lvs --noheading -o lv_name,vg_name,lv_path,snap_percent",
+    "GetMountCommand": "grep %s /etc/mtab",
     "CreateSnapshotCommand": "echo 'no action'",
     "AvailableLocalSnapshotHostsCommand": "echo 127.0.0.1",
     "AvailableSnapshotHostsCommand": "echo localhost\n127.0.0.1",

docker/entrypoint.sh

@@ -1,25 +1,34 @@
 #!/bin/sh
 if [ ! -e /etc/orchestrator-agent.conf.json ] ; then
+# @TODO think about full customization of orchestrator-agent.conf.json via ENV variables
+# @TODO think about mysql running not inside this docker
 cat <<EOF > /etc/orchestrator-agent.conf.json
 {
-    "SnapshotMountPoint": "/tmp",
-    "AgentsServer": "http://localhost",
-    "AgentsServerPort": ":3001",
-    "ContinuousPollSeconds" : 60,
-    "ResubmitAgentIntervalMinutes": 60,
-    "CreateSnapshotCommand": "echo 'no action'",
-    "AvailableLocalSnapshotHostsCommand": "echo 127.0.0.1",
-    "AvailableSnapshotHostsCommand": "echo localhost\n127.0.0.1",
-    "SnapshotVolumesFilter": "-my-snapshot-",
-    "MySQLDatadirCommand": "echo '~/tmp'",
+    "SnapshotMountPoint": "${SnapshotMountPoint:-/mysql-data}",
+    "AgentsServer": "${AgentsServer:-http://localhost}",
+    "AgentsServerPort": "${AgentsServerPort:-:3001}",
+    "ContinuousPollSeconds" : ${ContinuousPollSeconds:-60},
+    "ResubmitAgentIntervalMinutes": ${ResubmitAgentIntervalMinutes:-10},
+    "LogicalVolumesCommand": "${LogicalVolumesCommand:-lvs --noheading -o lv_name,vg_name,lv_path,snap_percent}",
+    "GetMountCommand":       "${GetMountCommand:-grep %s /etc/mtab}",
+    "UnmountCommand":        "${UnmountCommand:-umount %s}",
+    "MountLVCommand":        "${MountLVCommand:-mount %s %s %s}",
+    "RemoveLVCommand":       "${RemoveLVCommand:-lvremove --force %s}",
+    "MySQLTailErrorLogCommand": "${MySQLTailErrorLogCommand:-tail -n 20 \$(mysql -B --skip-column-names -e \"SELECT @@log_error\")}",
+    "GetLogicalVolumeFSTypeCommand": "${GetLogicalVolumeFSTypeCommand:-blkid %s}",
+    "CreateSnapshotCommand": "${CreateSnapshotCommand:-echo \'no action\'}",
+    "AvailableLocalSnapshotHostsCommand": "${AvailableLocalSnapshotHostsCommand:-echo 127.0.0.1}",
+    "AvailableSnapshotHostsCommand": "${AvailableSnapshotHostsCommand:-printf \'localhost\n127.0.0.1\'}",
+    "SnapshotVolumesFilter": "${SnapshotVolumesFilter:--mysql-snapshot-}",
+    "MySQLDatadirCommand": "mysql -B --skip-column-names -e 'SELECT @@datadir'",
     "MySQLPortCommand": "echo '3306'",
     "MySQLDeleteDatadirContentCommand": "echo 'will not do'",
     "MySQLServiceStopCommand":      "/etc/init.d/mysqld stop",
     "MySQLServiceStartCommand":     "/etc/init.d/mysqld start",
     "MySQLServiceStatusCommand":    "/etc/init.d/mysqld status",
-    "ReceiveSeedDataCommand":       "echo 'not implemented here'",
-    "SendSeedDataCommand":          "echo 'not implemented here'",
-    "PostCopyCommand":              "echo 'post copy'",
+    "ReceiveSeedDataCommand":       "${ReceiveSeedDataCommand:-echo \'not implemented here\'}",
+    "SendSeedDataCommand":          "${SendSeedDataCommand:-echo \'not implemented here\'}",
+    "PostCopyCommand":              "${PostCopyCommand:-echo \'post copy\'}",
     "HTTPPort": 3002,
     "HTTPAuthUser": "",
     "HTTPAuthPassword": "",

etc/init.d/orchestrator-agent.bash

@@ -26,6 +26,11 @@ DESC="orchestrator-agent: MySQL management agent"
 PIDFILE=/var/run/$NAME.pid
 SCRIPTNAME=/etc/init.d/$NAME
 
+# This files can be used to inject pre-service execution
+# scripts, such as exporting variables or whatever. It's yours!
+[ -f /etc/default/orchestrator-agent ] && . /etc/default/orchestrator-agent
+[ -f /etc/orchestrator-agent_profile ] && . /etc/orchestrator-agent_profile
+
 case "$1" in
   start)
     printf "%-50s" "Starting $NAME..."
@@ -60,21 +65,39 @@ case "$1" in
     PID=$(cat $PIDFILE)
     cd $DAEMON_PATH
     if [ -f $PIDFILE ]; then
-      kill -HUP $PID
-      printf "%s\n" "Ok"
+      kill -TERM $PID
       rm -f $PIDFILE
+      # Wait for orchestrator to stop otherwise restart may fail.
+      # (The newly restarted process may be unable to bind to the
+      # currently bound socket.)
+      while ps -p $PID >/dev/null 2>&1; do
+        printf "."
+        sleep 1
+      done
+      printf "\n"
+      printf "Ok\n"
     else
       printf "%s\n" "pidfile not found"
       exit 1
     fi
   ;;
-
   restart)
     $0 stop
     $0 start
   ;;
-
+  reload)
+    printf "%-50s" "Reloading $NAME"
+    PID=$(cat $PIDFILE)
+    cd $DAEMON_PATH
+    if [ -f $PIDFILE ]; then
+      kill -HUP $PID
+      printf "%s\n" "Ok"
+    else
+      printf "%s\n" "pidfile not found"
+      exit 1
+    fi
+	;;
   *)
-    echo "Usage: $0 {status|start|stop|restart}"
+    echo "Usage: $0 {status|start|stop|restart|reload}"
     exit 1
 esac

go/cmd/orchestrator-agent/main.go

@@ -33,11 +33,22 @@ var AppVersion string
 
 func acceptSignal() {
 	c := make(chan os.Signal, 1)
-	signal.Notify(c, os.Interrupt, os.Kill, syscall.SIGHUP)
 
 	// Block until a signal is received.
-	sig := <-c
-	log.Fatalf("Got signal: %+v", sig)
+	signal.Notify(c, syscall.SIGHUP, syscall.SIGKILL, syscall.SIGTERM, syscall.SIGTERM)
+	go func() {
+		for sig := range c {
+			switch sig {
+			case syscall.SIGHUP:
+				log.Infof("Received SIGHUP. Reloading configuration")
+				config.Reload()
+			case syscall.SIGTERM, syscall.SIGKILL, syscall.SIGINT:
+				log.Infof("Received %s. Shutting down orchestrator-agent", sig.String())
+				// probably should poke other go routines to stop cleanly here ...
+				os.Exit(0)
+			}
+		}
+	}()
 }
 
 // main is the application's entry point. It will either spawn a CLI or HTTP itnerfaces.

go/config/config.go

@@ -28,6 +28,13 @@ type Configuration struct {
 	SnapshotMountPoint                 string            // The single, agreed-upon mountpoint for logical volume snapshots
 	ContinuousPollSeconds              uint              // Poll interval for continuous operation
 	ResubmitAgentIntervalMinutes       uint              // Poll interval for resubmitting this agent on orchestrator agents API
+	LogicalVolumesCommand              string            // Command which list logical volumes, default implementation used lvs
+	GetMountCommand                    string            // Command which get mount point parameters by mount point name, default implementation over cat %s/etc/mtab
+	UnmountCommand                     string            // Command which unmount current mount point, default implementation just execute `umount %s`
+	MountLVCommand                     string            // Command which mount selected snapshot into mount point
+	RemoveLVCommand                    string            // Command which remove selected snapshot from disk
+	MySQLTailErrorLogCommand           string            // Command which return last 20 lines from @@log_error file
+	GetLogicalVolumeFSTypeCommand      string            // Command which return logical volume filesystem type
 	CreateSnapshotCommand              string            // Command which creates a snapshot logical volume. It's a "do it yourself" implementation
 	AvailableLocalSnapshotHostsCommand string            // Command which returns list of hosts (one host per line) with available snapshots in local datacenter
 	AvailableSnapshotHostsCommand      string            // Command which returns list of hosts (one host per line) with available snapshots in any datacenter
@@ -45,6 +52,7 @@ type Configuration struct {
 	AgentsServer                       string            // HTTP address of the orchestrator agents server
 	AgentsServerPort                   string            // HTTP port of the orchestrator agents server
 	HTTPPort                           uint              // HTTP port on which this service listens
+	SeedTransferPort                   uint              // TCP port for data seed transfer
 	HTTPAuthUser                       string            // Username for HTTP Basic authentication (blank disables authentication)
 	HTTPAuthPassword                   string            // Password for HTTP Basic authentication
 	UseSSL                             bool              // If true, service will serve HTTPS only
@@ -65,13 +73,21 @@ type Configuration struct {
 }
 
 var Config = NewConfiguration()
+var configFileNames []string
 
 func NewConfiguration() *Configuration {
 	return &Configuration{
 		SnapshotMountPoint:                 "",
 		ContinuousPollSeconds:              60,
 		ResubmitAgentIntervalMinutes:       60,
 		CreateSnapshotCommand:              "",
+		LogicalVolumesCommand:              "lvs --noheading -o lv_name,vg_name,lv_path,snap_percent",
+		GetMountCommand:                    "grep %s /etc/mtab",
+		UnmountCommand:                     "umount %s",
+		MountLVCommand:                     "mount %s %s %s",
+		RemoveLVCommand:                    "lvremove --force %s",
+		MySQLTailErrorLogCommand:           `tail -n 20 $(egrep "log[-_]error" /etc/my.cnf | cut -d "=" -f 2)`,
+		GetLogicalVolumeFSTypeCommand:      "blkid %s",
 		AvailableLocalSnapshotHostsCommand: "",
 		AvailableSnapshotHostsCommand:      "",
 		SnapshotVolumesFilter:              "",
@@ -88,6 +104,7 @@ func NewConfiguration() *Configuration {
 		AgentsServer:                       "",
 		AgentsServerPort:                   "",
 		HTTPPort:                           3002,
+		SeedTransferPort:                   21234,
 		HTTPAuthUser:                       "",
 		HTTPAuthPassword:                   "",
 		UseSSL:                             false,
@@ -130,9 +147,15 @@ func Read(file_names ...string) *Configuration {
 	for _, file_name := range file_names {
 		read(file_name)
 	}
+	configFileNames = file_names
 	return Config
 }
 
+// Reload
+func Reload() *Configuration {
+	return Read(configFileNames...)
+}
+
 // ForceRead reads configuration from given file name or bails out if it fails
 func ForceRead(file_name string) *Configuration {
 	_, err := read(file_name)

go/http/api.go

@@ -363,7 +363,7 @@ func (this *HttpAPI) PostCopy(params martini.Params, r render.Render, req *http.
 	if err := this.validateToken(r, req); err != nil {
 		return
 	}
-	err := osagent.PostCopy()
+	err := osagent.PostCopy(params["sourceHost"])
 	if err != nil {
 		r.JSON(500, &APIResponse{Code: ERROR, Message: err.Error()})
 		return
@@ -617,6 +617,7 @@ func (this *HttpAPI) RegisterRequests(m *martini.ClassicMartini) {
 	m.Get("/api/delete-mysql-datadir", this.DeleteMySQLDataDir)
 	m.Get("/api/mysql-datadir-available-space", this.GetMySQLDataDirAvailableDiskSpace)
 	m.Get("/api/post-copy", this.PostCopy)
+	m.Get("/api/post-copy/:sourceHost", this.PostCopy)
 	m.Get("/api/receive-mysql-seed-data/:seedId", this.ReceiveMySQLSeedData)
 	m.Get("/api/send-mysql-seed-data/:targetHost/:seedId", this.SendMySQLSeedData)
 	m.Get("/api/abort-seed/:seedId", this.AbortSeed)