Add kubernetes election module (#3721)

Introduce a new election module using the Kubernetes Lease API. Modify
the existing mechanism to allow selection of election mechanisms by
introducing the `--election_system` parameter, following the same
pattern used for storage and quota system selection.

---------

Co-authored-by: Martin Hutchinson <mhutchinson@gmail.com>

Author: osmman

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ## HEAD
 
+* Add `--election_system` parameter to allow selection of the election system by @osmman in https://github.com/google/trillian/pull/3721
+* Add Kubernetes-based leader election system by @osmman in https://github.com/google/trillian/pull/3721
+
 ## v1.7.1
 
 ### Storage

CONTRIBUTORS

@@ -37,4 +37,5 @@ Pierre Phaneuf <pphaneuf@google.com> <pphaneuf@gmail.com>
 Rob Percival <robpercival@google.com>
 Rob Stradling <rob@sectigo.com>
 Roger Ng <rogerng@google.com> <roger2hk@gmail.com>
+Tomas Turek <tturek@redhat.com>
 Vishal Kuo <vishalkuo@gmail.com>

cmd/internal/provider/default_systems.go

@@ -5,11 +5,13 @@ import (
 
 	"github.com/google/trillian/quota"
 	"github.com/google/trillian/storage"
+	"github.com/google/trillian/util/election2"
 )
 
 var (
-	DefaultQuotaSystem   string
-	DefaultStorageSystem string
+	DefaultQuotaSystem    string
+	DefaultStorageSystem  string
+	DefaultElectionSystem string
 )
 
 func init() {
@@ -27,4 +29,12 @@ func init() {
 		defaultProvider = providers[0]
 	}
 	DefaultQuotaSystem = defaultProvider
+
+	defaultProvider = "etcd"
+	providers = election2.Providers()
+	if len(providers) > 0 && !slices.Contains(providers, defaultProvider) {
+		slices.Sort(providers)
+		defaultProvider = providers[0]
+	}
+	DefaultElectionSystem = defaultProvider
 }

cmd/internal/provider/etcd.go

@@ -0,0 +1,7 @@
+//go:build etcd || !k8s
+
+package provider
+
+import (
+	_ "github.com/google/trillian/util/election2/etcd"
+)

cmd/internal/provider/k8s.go

@@ -0,0 +1,7 @@
+//go:build k8s || !etcd
+
+package provider
+
+import (
+	_ "github.com/google/trillian/util/election2/k8s"
+)

cmd/trillian_log_signer/main.go

@@ -46,7 +46,6 @@ import (
 	"github.com/google/trillian/util/clock"
 	"github.com/google/trillian/util/election"
 	"github.com/google/trillian/util/election2"
-	etcdelect "github.com/google/trillian/util/election2/etcd"
 	clientv3 "go.etcd.io/etcd/client/v3"
 	"google.golang.org/grpc"
 	"k8s.io/klog/v2"
@@ -66,7 +65,6 @@ var (
 	sequencerGuardWindowFlag = flag.Duration("sequencer_guard_window", 0, "If set, the time elapsed before submitted leaves are eligible for sequencing")
 	forceMaster              = flag.Bool("force_master", false, "If true, assume master for all logs")
 	etcdHTTPService          = flag.String("etcd_http_service", "trillian-logsigner-http", "Service name to announce our HTTP endpoint under")
-	lockDir                  = flag.String("lock_file_path", "/test/multimaster", "etcd lock file directory path")
 	healthzTimeout           = flag.Duration("healthz_timeout", time.Second*5, "Timeout used during healthz checks")
 
 	quotaSystem         = flag.String("quota_system", provider.DefaultQuotaSystem, fmt.Sprintf("Quota system to use. One of: %v", quota.Providers()))
@@ -76,6 +74,7 @@ var (
 
 	storageSystem = flag.String("storage_system", provider.DefaultStorageSystem, fmt.Sprintf("Storage system to use. One of: %v", storage.Providers()))
 
+	electionSystem     = flag.String("election_system", provider.DefaultElectionSystem, fmt.Sprintf("Election system to use. One of: %v", election2.Providers()))
 	preElectionPause   = flag.Duration("pre_election_pause", 1*time.Second, "Maximum time to wait before starting elections")
 	masterHoldInterval = flag.Duration("master_hold_interval", 60*time.Second, "Minimum interval to hold mastership for")
 	masterHoldJitter   = flag.Duration("master_hold_jitter", 120*time.Second, "Maximal random addition to --master_hold_interval")
@@ -133,17 +132,16 @@ func main() {
 	defer cancel()
 	go util.AwaitSignal(ctx, cancel)
 
-	hostname, _ := os.Hostname()
-	instanceID := fmt.Sprintf("%s.%d", hostname, os.Getpid())
 	var electionFactory election2.Factory
 	switch {
 	case *forceMaster:
 		klog.Warning("**** Acting as master for all logs ****")
 		electionFactory = election2.NoopFactory{}
-	case client != nil:
-		electionFactory = etcdelect.NewFactory(instanceID, client, *lockDir)
 	default:
-		klog.Exit("Either --force_master or --etcd_servers must be supplied")
+		electionFactory, err = election2.NewProvider(*electionSystem)
+		if err != nil {
+			klog.Exitf("Failed to get election provider: %v", err)
+		}
 	}
 
 	qm, err := quota.NewManager(*quotaSystem)

docs/Feature_Implementation_Matrix.md

@@ -109,10 +109,11 @@ Supported monitoring frameworks, allowing for production monitoring and alerting
 
 Supported frameworks for providing Master Election.
 
-| Election        | Status  | Deployed in prod    | Notes                                                                       |
-|:---             | :---:   | :---:               |:---                                                                         |
-| Chubby          | GA      | ✓                   | Google internal-only.                                                       |
-| etcd            | GA      | ✓                   |                                                                             |
+| Election | Status | Deployed in prod | Notes                                                                                  |
+|:---------|:------:|:----------------:|:---------------------------------------------------------------------------------------|
+| Chubby   |   GA   |        ✓         | Google internal-only.                                                                  |
+| etcd     |   GA   |        ✓         |                                                                                        |
+| k8s      | Alpha  |                  | Supported by [Tomas Turek](https://github.com/osmman) at [Red Hat](https://redhat.com) |
 
 ### Quota
 

go.mod

@@ -43,7 +43,11 @@ require (
 	google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.5.1
 	google.golang.org/protobuf v1.36.3
 	gopkg.in/yaml.v2 v2.4.0
+	k8s.io/api v0.31.5
+	k8s.io/apimachinery v0.31.5
+	k8s.io/client-go v0.31.5
 	k8s.io/klog/v2 v2.130.1
+	k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738
 )
 
 require (
@@ -85,16 +89,23 @@ require (
 	github.com/docker/go-connections v0.5.0 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/envoyproxy/go-control-plane v0.13.1 // indirect
 	github.com/envoyproxy/protoc-gen-validate v1.1.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-openapi/jsonpointer v0.20.2 // indirect
+	github.com/go-openapi/jsonreference v0.20.4 // indirect
+	github.com/go-openapi/swag v0.22.9 // indirect
 	github.com/gofrs/flock v0.8.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt/v4 v4.5.1 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
+	github.com/google/gnostic-models v0.6.8 // indirect
+	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/licenseclassifier/v2 v2.0.0 // indirect
 	github.com/google/pprof v0.0.0-20240528025155-186aa0362fba // indirect
 	github.com/google/s2a-go v0.1.9 // indirect
@@ -120,8 +131,10 @@ require (
 	github.com/jhump/protoreflect v1.16.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.1-0.20220621161143-b0104c826a24 // indirect
 	github.com/jonboulle/clockwork v0.4.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.17.11 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-runewidth v0.0.13 // indirect
 	github.com/miekg/pkcs11 v1.1.1 // indirect
 	github.com/mitchellh/copystructure v1.0.0 // indirect
@@ -152,6 +165,7 @@ require (
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 // indirect
 	github.com/urfave/cli v1.22.14 // indirect
+	github.com/x448/float16 v0.8.4 // indirect
 	github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 // indirect
 	go.etcd.io/bbolt v1.3.11 // indirect
 	go.etcd.io/etcd/api/v3 v3.5.17 // indirect
@@ -178,10 +192,16 @@ require (
 	golang.org/x/mod v0.22.0 // indirect
 	golang.org/x/net v0.34.0 // indirect
 	golang.org/x/oauth2 v0.25.0 // indirect
+	golang.org/x/term v0.28.0 // indirect
 	golang.org/x/text v0.21.0 // indirect
 	golang.org/x/time v0.9.0 // indirect
 	gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect
+	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
+	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
 	sigs.k8s.io/yaml v1.4.0 // indirect
 )