Merge pull request #250 from SeanHai/curve-manager

Support manage curve monitor, curve website and support http service

Author: SeanHai

.github/workflows/go.yml

@@ -14,7 +14,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.18
+          go-version: 1.19
       - name: Generate SSH key
         run: ssh-keygen -f ~/.ssh/id_rsa -P ""
       - name: Test
@@ -28,7 +28,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.18
+          go-version: 1.19
       - name: Set dependencies
         run: sudo apt update && sudo apt install musl-tools
       - name: Build

Makefile

@@ -20,6 +20,7 @@ GO := go
 
 # output
 OUTPUT := bin/curveadm
+SERVER_OUTPUT := bin/pigeon
 
 # build flags
 LDFLAGS := -s -w
@@ -50,15 +51,18 @@ TEST_FLAGS += -run $(CASE)
 
 # packages
 PACKAGES := $(PWD)/cmd/curveadm/main.go
+SERVER_PACKAGES := $(PWD)/cmd/service/main.go
 
 # tar
 VERSION := "unknown"
 
 build:
 	$(GOENV) $(GO) build -o $(OUTPUT) $(BUILD_FLAGS) $(PACKAGES)
+	$(GOENV) $(GO) build -o $(SERVER_OUTPUT) $(BUILD_FLAGS) $(SERVER_PACKAGES)
 
 debug:
 	$(GOENV) $(GO) build -o $(OUTPUT) $(DEBUG_FLAGS) $(PACKAGES)
+	$(GOENV) $(GO) build -o $(SERVER_OUTPUT) $(DEBUG_FLAGS) $(SERVER_PACKAGES)
 
 test:
 	$(GO_TEST) $(TEST_FLAGS) ./...

build/package/upload.sh

@@ -5,10 +5,12 @@ g_nos_cmd=${NOSCMD}
 g_root=$(pwd)/.build
 g_curveadm=${g_root}/curveadm
 g_curveadm_bin=${g_curveadm}/bin
+g_curveadm_http_bin=${g_curveadm}/http
 rm -rf ${g_root}
 
-mkdir -p ${g_curveadm_bin}
+mkdir -p ${g_curveadm_bin} ${g_curveadm_http_bin}
 cp bin/curveadm ${g_curveadm_bin}
+cp bin/pigeon ${g_curveadm_http_bin}
 [[ -f .CHANGELOG ]] && cp .CHANGELOG ${g_curveadm}/CHANGELOG
 (cd ${g_curveadm} && ./bin/curveadm -v && ls -ls bin/curveadm && [[ -f CHANGELOG ]] && cat CHANGELOG)
 (cd ${g_root} && tar -zcf curveadm-${g_version}.tar.gz curveadm)

cli/cli/cli.go

@@ -72,6 +72,7 @@ type CurveAdm struct {
 	clusterName         string         // current cluster name
 	clusterTopologyData string         // cluster topology
 	clusterPoolData     string         // cluster pool
+	monitor             storage.Monitor
 }
 
 /*
@@ -193,6 +194,13 @@ func (curveadm *CurveAdm) init() error {
 		return errno.ERR_GET_DISK_RECORDS_FAILED.E(err)
 	}
 
+	// (10) Get monitor configure
+	monitor, err := s.GetMonitor(cluster.Id)
+	if err != nil {
+		log.Error("Get monitor failed", log.Field("Error", err))
+		return errno.ERR_GET_MONITOR_FAILED.E(err)
+	}
+
 	curveadm.dbpath = dbpath
 	curveadm.logpath = logpath
 	curveadm.config = config
@@ -209,6 +217,7 @@ func (curveadm *CurveAdm) init() error {
 	curveadm.clusterName = cluster.Name
 	curveadm.clusterTopologyData = cluster.Topology
 	curveadm.clusterPoolData = cluster.Pool
+	curveadm.monitor = monitor
 
 	return nil
 }
@@ -292,6 +301,7 @@ func (curveadm *CurveAdm) ClusterUUId() string               { return curveadm.c
 func (curveadm *CurveAdm) ClusterName() string               { return curveadm.clusterName }
 func (curveadm *CurveAdm) ClusterTopologyData() string       { return curveadm.clusterTopologyData }
 func (curveadm *CurveAdm) ClusterPoolData() string           { return curveadm.clusterPoolData }
+func (curveadm *CurveAdm) Monitor() storage.Monitor          { return curveadm.monitor }
 
 func (curveadm *CurveAdm) GetHost(host string) (*hosts.HostConfig, error) {
 	if len(curveadm.Hosts()) == 0 {
@@ -367,6 +377,11 @@ func (curveadm *CurveAdm) GetServiceId(dcId string) string {
 	return utils.MD5Sum(serviceId)[:12]
 }
 
+// website is cluster-independent
+func (curveadm *CurveAdm) GetWebsiteServiceId(wcId string) string {
+	return utils.MD5Sum(wcId)[:12]
+}
+
 func (curveadm *CurveAdm) GetContainerId(serviceId string) (string, error) {
 	containerId, err := curveadm.Storage().GetContainerId(serviceId)
 	if err != nil {

cli/command/cluster/add.go

@@ -37,9 +37,9 @@ import (
 
 const (
 	ADD_EXAMPLE = `Examples:
-  $ curveadm add my-cluster                            # Add a cluster named 'my-cluster'
-  $ curveadm add my-cluster -m "deploy for test"       # Add a cluster with description
-  $ curveadm add my-cluster -f /path/to/topology.yaml  # Add a cluster with specified topology`
+  $ curveadm cluster add my-cluster                            # Add a cluster named 'my-cluster'
+  $ curveadm cluster add my-cluster -m "deploy for test"       # Add a cluster with description
+  $ curveadm cluster add my-cluster -f /path/to/topology.yaml  # Add a cluster with specified topology`
 )
 
 var (
@@ -92,8 +92,7 @@ func readTopology(filename string) (string, error) {
 }
 
 func genCheckTopologyPlaybook(curveadm *cli.CurveAdm,
-	dcs []*topology.DeployConfig,
-	options addOptions) (*playbook.Playbook, error) {
+	dcs []*topology.DeployConfig) (*playbook.Playbook, error) {
 	steps := CHECK_TOPOLOGY_PLAYBOOK_STEPS
 	pb := playbook.NewPlaybook(curveadm)
 	for _, step := range steps {
@@ -116,8 +115,8 @@ func genCheckTopologyPlaybook(curveadm *cli.CurveAdm,
 	return pb, nil
 }
 
-func checkTopology(curveadm *cli.CurveAdm, data string, options addOptions) error {
-	if len(options.filename) == 0 {
+func checkTopology(curveadm *cli.CurveAdm, data string, filename string) error {
+	if len(filename) == 0 {
 		return nil
 	}
 
@@ -126,7 +125,7 @@ func checkTopology(curveadm *cli.CurveAdm, data string, options addOptions) erro
 		return err
 	}
 
-	pb, err := genCheckTopologyPlaybook(curveadm, dcs, options)
+	pb, err := genCheckTopologyPlaybook(curveadm, dcs)
 	if err != nil {
 		return err
 	}
@@ -155,7 +154,7 @@ func runAdd(curveadm *cli.CurveAdm, options addOptions) error {
 	}
 
 	// 3) check topology
-	err = checkTopology(curveadm, data, options)
+	err = checkTopology(curveadm, data, options.filename)
 	if err != nil {
 		return err
 	}
@@ -170,3 +169,28 @@ func runAdd(curveadm *cli.CurveAdm, options addOptions) error {
 	curveadm.WriteOutln("Added cluster '%s'", name)
 	return nil
 }
+
+// for http service
+func Add(curveadm *cli.CurveAdm, name, desc, topo string) error {
+	storage := curveadm.Storage()
+	clusters, err := storage.GetClusters(name)
+	if err != nil {
+		return errno.ERR_GET_ALL_CLUSTERS_FAILED.E(err)
+	} else if len(clusters) > 0 {
+		return errno.ERR_CLUSTER_ALREADY_EXIST.
+			F("cluster name: %s", name)
+	}
+
+	// check topology
+	err = checkTopology(curveadm, topo, topo)
+	if err != nil {
+		return err
+	}
+
+	// 4) insert cluster (with topology) into database
+	err = storage.InsertCluster(name, desc, topo)
+	if err != nil {
+		return errno.ERR_INSERT_CLUSTER_FAILED.E(err)
+	}
+	return nil
+}

cli/command/cluster/checkout.go

@@ -77,3 +77,22 @@ func runCheckout(curveadm *cli.CurveAdm, options checkoutOptions) error {
 	curveadm.WriteOutln("Switched to cluster '%s'", clusterName)
 	return nil
 }
+
+// for http service
+func Checkout(curveadm *cli.CurveAdm, clusterName string) error {
+	storage := curveadm.Storage()
+	clusters, err := storage.GetClusters(clusterName)
+	if err != nil {
+		return errno.ERR_GET_ALL_CLUSTERS_FAILED.E(err)
+	} else if len(clusters) == 0 {
+		return errno.ERR_CLUSTER_NOT_FOUND.
+			F("cluster name: %s", clusterName)
+	}
+
+	// 2) switch current cluster in database
+	err = storage.CheckoutCluster(clusterName)
+	if err != nil {
+		return errno.ERR_CHECKOUT_CLUSTER_FAILED.E(err)
+	}
+	return nil
+}

cli/command/cluster/list.go

@@ -27,6 +27,7 @@ package cluster
 import (
 	"github.com/opencurve/curveadm/cli/cli"
 	"github.com/opencurve/curveadm/internal/errno"
+	"github.com/opencurve/curveadm/internal/storage"
 	"github.com/opencurve/curveadm/internal/tui"
 	cliutil "github.com/opencurve/curveadm/internal/utils"
 	log "github.com/opencurve/curveadm/pkg/log/glg"
@@ -72,3 +73,13 @@ func runList(curveadm *cli.CurveAdm, options listOptions) error {
 	curveadm.WriteOut(output)
 	return nil
 }
+
+// for http service
+func List(curveadm *cli.CurveAdm) ([]storage.Cluster, error) {
+	storage := curveadm.Storage()
+	clusters, err := storage.GetClusters("%")
+	if err != nil {
+		return nil, errno.ERR_GET_ALL_CLUSTERS_FAILED.E(err)
+	}
+	return clusters, nil
+}

cli/command/cmd.go

@@ -33,9 +33,12 @@ import (
 	"github.com/opencurve/curveadm/cli/command/config"
 	"github.com/opencurve/curveadm/cli/command/disks"
 	"github.com/opencurve/curveadm/cli/command/hosts"
+	"github.com/opencurve/curveadm/cli/command/http"
+	"github.com/opencurve/curveadm/cli/command/monitor"
 	"github.com/opencurve/curveadm/cli/command/pfs"
 	"github.com/opencurve/curveadm/cli/command/playground"
 	"github.com/opencurve/curveadm/cli/command/target"
+	"github.com/opencurve/curveadm/cli/command/website"
 	"github.com/opencurve/curveadm/internal/errno"
 	tools "github.com/opencurve/curveadm/internal/tools/upgrade"
 	cliutil "github.com/opencurve/curveadm/internal/utils"
@@ -66,6 +69,9 @@ func addSubCommands(cmd *cobra.Command, curveadm *cli.CurveAdm) {
 		playground.NewPlaygroundCommand(curveadm), // curveadm playground ...
 		target.NewTargetCommand(curveadm),         // curveadm target ...
 		pfs.NewPFSCommand(curveadm),               // curveadm pfs ...
+		monitor.NewMonitorCommand(curveadm),       // curveadm monitor ...
+		http.NewHttpCommand(curveadm),             // curveadm http
+		website.NewWebsiteCommand(curveadm),       // curveadm website ...
 
 		NewAuditCommand(curveadm),      // curveadm audit
 		NewCleanCommand(curveadm),      // curveadm clean

cli/command/config/commit.go

@@ -105,8 +105,7 @@ func checkDiff(curveadm *cli.CurveAdm, newData string) error {
 }
 
 func genCheckTopologyPlaybook(curveadm *cli.CurveAdm,
-	dcs []*topology.DeployConfig,
-	options commitOptions) (*playbook.Playbook, error) {
+	dcs []*topology.DeployConfig) (*playbook.Playbook, error) {
 	steps := CHECK_TOPOLOGY_PLAYBOOK_STEPS
 	pb := playbook.NewPlaybook(curveadm)
 	for _, step := range steps {
@@ -161,7 +160,7 @@ func checkTopology(curveadm *cli.CurveAdm, data string, options commitOptions) e
 		return err
 	}
 
-	pb, err := genCheckTopologyPlaybook(curveadm, dcs, options)
+	pb, err := genCheckTopologyPlaybook(curveadm, dcs)
 	if err != nil {
 		return err
 	}
@@ -217,3 +216,25 @@ func runCommit(curveadm *cli.CurveAdm, options commitOptions) error {
 	curveadm.WriteOutln("Cluster '%s' topology updated", curveadm.ClusterName())
 	return err
 }
+
+// for http service
+func Commit(curveadm *cli.CurveAdm, name, conf string) error {
+	// parse cluster topology
+	_, err := curveadm.ParseTopology()
+	if err != nil && !skipError(err) {
+		return err
+	}
+
+	// check topology
+	err = checkTopology(curveadm, conf, commitOptions{force: false})
+	if err != nil {
+		return err
+	}
+
+	// update cluster topology in database
+	err = curveadm.Storage().SetClusterTopologyByName(name, conf)
+	if err != nil {
+		return errno.ERR_UPDATE_CLUSTER_TOPOLOGY_FAILED.E(err)
+	}
+	return err
+}

cli/command/config/show.go

@@ -97,3 +97,12 @@ func runShow(curveadm *cli.CurveAdm, options showOptions) error {
 	curveadm.WriteOutln(data)
 	return nil
 }
+
+// for http service
+func Show(curveadm *cli.CurveAdm) (string, string, error) {
+	cluster, err := curveadm.Storage().GetCurrentCluster()
+	if err != nil {
+		return "", "", errno.ERR_GET_CURRENT_CLUSTER_FAILED.E(err)
+	}
+	return cluster.Name, cluster.Topology, nil
+}