Add additional fields to the model for the FDB database status. (#2267)

Author: brownleej

api/v1beta2/foundationdb_status.go

@@ -140,6 +140,21 @@ type FoundationDBStatusClusterInfo struct {
 
 	// BounceImpact represents the bounce_impact part of the machine-readable status.
 	BounceImpact FoundationDBBounceImpact `json:"bounce_impact,omitempty"`
+
+	// DatabaseAvailable indicates whether the database is currently available.
+	DatabaseAvailable *bool `json:"database_available,omitempty"`
+
+	// ActivePrimaryDC indicates which data center is currently the primary.
+	ActivePrimaryDC *string `json:"active_primary_dc,omitempty"`
+
+	// DatabaseLockState provides information about the database's locking.
+	DatabaseLockState FoundationDBStatusLockState `json:"database_lock_state,omitempty"`
+}
+
+// FoundationDBStatusLockState provides information about the database's locking.
+type FoundationDBStatusLockState struct {
+	// Locked indicates whether the database is currently locked.
+	Locked *bool `json:"locked,omitempty"`
 }
 
 // FoundationDBBounceImpact represents the bounce_impact part of the machine-readable status.
@@ -209,6 +224,51 @@ type FoundationDBStatusProcessRoleInfo struct {
 	StoredBytes int `json:"stored_bytes,omitempty"`
 	// ID represent the role ID.
 	ID string `json:"id,omitempty"`
+
+	// DataLag indicates whether this process is lagging in its writes.
+	DataLag FoundationDBStatusLagInfo `json:"data_lag,omitempty"`
+
+	// KVStoreUsedBytes indicates how much space this process is using on its disk.
+	KVStoreUsedBytes *int64 `json:"kvstore_used_bytes"`
+
+	// KVStoreTotalBytes indicates how much space this process's disk has in total.
+	KVStoreTotalBytes *int64 `json:"kvstore_total_bytes"`
+
+	// KVStoreFreeBytes indicates how much free space this process's disk has.
+	KVStoreFreeBytes *int64 `json:"kvstore_free_bytes"`
+
+	// KVStoreAvailableBytes indicates how much space this process can still use on its disk, inclusive of both free space and unused written space.
+	KVStoreAvailableBytes *int64 `json:"kvstore_available_bytes"`
+
+	// ReadLatencyStatistics provides statistics about this process's read latencies.
+	ReadLatencyStatistics FoundationDBStatusPerfStatistics `json:"read_latency_statistics"`
+
+	// ReadLatencyStatistics provides statistics about this process's commit latencies.
+	CommitLatencyStatistics FoundationDBStatusPerfStatistics `json:"commit_latency_statistics"`
+
+	// ReadLatencyStatistics provides statistics about this process's GRV latencies.
+	GRVLatencyStatistics FoundationDBStatusGRVStatistics `json:"grv_latency_statistics"`
+}
+
+// FoundationDBStatusPerfStatistics models information about one dimension of a process's performance.
+type FoundationDBStatusPerfStatistics struct {
+	// Count provides the number of observations for this stat.
+	Count *int `json:"count"`
+
+	// Median provides the median value for this stat.
+	Median *float64 `json:"median"`
+
+	// P99 provides the 99th percentile value for this stat.
+	P99 *float64 `json:"p99"`
+}
+
+// FoundationDBStatusGRVStatistics models information about a process's GRV latencies.
+type FoundationDBStatusGRVStatistics struct {
+	// Batch provides the GRV latencies at batch priority.
+	Batch FoundationDBStatusPerfStatistics `json:"batch"`
+
+	// Default provides the GRV latencies at default priority.
+	Default FoundationDBStatusPerfStatistics `json:"default"`
 }
 
 // FoundationDBStatusDataStatistics provides information about the data in
@@ -225,6 +285,9 @@ type FoundationDBStatusDataStatistics struct {
 
 	// TeamTrackers provides information about (highest priority) teams.
 	TeamTrackers []FoundationDBStatusTeamTracker `json:"team_trackers,omitempty"`
+
+	// TotalDiskUsedBytes provides the total bytes used on disk by the database.
+	TotalDiskUsedBytes *int64 `json:"total_disk_used_bytes,omitempty"`
 }
 
 // FoundationDBStatusDataState provides information about the state of data
@@ -268,6 +331,9 @@ type FoundationDBStatusTeamTracker struct {
 	// FoundationDBStatusDataState provides information about the state of the
 	// highest priority team.
 	State FoundationDBStatusDataState `json:"state,omitempty"`
+
+	// UnhealthyServers defines how many unhealthy servers this team has.
+	UnhealthyServers *int64 `json:"unhealthy_servers,omitempty"`
 }
 
 // FoundationDBStatusClientDBStatus represents the databaseStatus field in the
@@ -335,6 +401,9 @@ func (client FoundationDBStatusConnectedClient) Description() string {
 // FoundationDBStatusLayerInfo provides information about layers that are
 // running against the cluster.
 type FoundationDBStatusLayerInfo struct {
+	// Valid indicates whether the layer info is valid, meaning that it can be trusted as a full set of information.
+	Valid *bool `json:"_valid,omitempty"`
+
 	// Backup provides information about backups that have been started.
 	Backup FoundationDBStatusBackupInfo `json:"backup,omitempty"`
 
@@ -354,9 +423,20 @@ type FoundationDBStatusBackupInfo struct {
 // FoundationDBStatusBackupTag provides information about a backup under a tag
 // in the cluster status.
 type FoundationDBStatusBackupTag struct {
+	// CurrentContainer identifies the container for this backup.
 	CurrentContainer string `json:"current_container,omitempty"`
-	RunningBackup    bool   `json:"running_backup,omitempty"`
-	Restorable       bool   `json:"running_backup_is_restorable,omitempty"`
+
+	// RunningBackup indicates whether the backup is currently running.
+	RunningBackup *bool `json:"running_backup,omitempty"`
+
+	// Restorable indicates whether the current backup is restorable.
+	Restorable *bool `json:"running_backup_is_restorable,omitempty"`
+
+	// LastRestorableSecondsBehind indicates how long ago the last restorable point-in-time for the backup is.
+	LastRestorableSecondsBehind *float64 `json:"last_restorable_seconds_behind,omitempty"`
+
+	// LastRestorableSecondsBehind indicates how many versions behind the last restorable point-in-time for the backup is.
+	LastRestorableVersion *int64 `json:"last_restorable_version,omitempty"`
 }
 
 // FoundationDBStatusLogInfo provides information about the fault tolerance metrics

api/v1beta2/foundationdb_status_test.go

@@ -22,20 +22,26 @@ package v1beta2
 
 import (
 	"encoding/json"
+	"maps"
 	"net"
 	"os"
 	"path/filepath"
-
-	"k8s.io/utils/pointer"
+	"slices"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	"github.com/onsi/gomega/format"
+	"k8s.io/utils/pointer"
 )
 
 var _ = Describe("FoundationDBStatus", func() {
+	BeforeEach(func() {
+		format.MaxLength = 10000000
+	})
+
 	When("parsing the status json with a 7.1.0-rc1 cluster", func() {
 		migrationType := StorageMigrationTypeDisabled
-		status := FoundationDBStatusClusterInfo{
+		clusterInfo := FoundationDBStatusClusterInfo{
 			Messages:                []FoundationDBStatusMessage{},
 			IncompatibleConnections: []string{},
 			ConnectionString:        "test_cluster:[email protected]:4501,10.1.18.254:4501,10.1.19.0:4501",
@@ -73,10 +79,23 @@ var _ = Describe("FoundationDBStatus", func() {
 						{
 							Role: string(ProcessRoleGrvProxy),
 							ID:   "0de7f5c5e549cad1",
+							GRVLatencyStatistics: FoundationDBStatusGRVStatistics{
+								Batch:   FoundationDBStatusPerfStatistics{Count: pointer.Int(6), Median: pointer.Float64(0.000531435), P99: pointer.Float64(0.00130677)},
+								Default: FoundationDBStatusPerfStatistics{Count: pointer.Int(225), Median: pointer.Float64(0.00062561), P99: pointer.Float64(0.010715200000000001)},
+							},
 						},
 						{
 							Role: string(ProcessRoleStorage),
 							ID:   "9941616400759d37",
+							DataLag: FoundationDBStatusLagInfo{
+								Seconds:  0.19625800000000002,
+								Versions: 196258,
+							},
+							KVStoreUsedBytes:      pointer.Int64(104878232),
+							KVStoreTotalBytes:     pointer.Int64(135012552704),
+							KVStoreFreeBytes:      pointer.Int64(84178223104),
+							KVStoreAvailableBytes: pointer.Int64(84178223104),
+							ReadLatencyStatistics: FoundationDBStatusPerfStatistics{Count: pointer.Int(297), Median: pointer.Float64(0.000116825), P99: pointer.Float64(0.000711441)},
 						},
 					},
 					Messages: []FoundationDBStatusProcessMessage{},
@@ -100,8 +119,14 @@ var _ = Describe("FoundationDBStatus", func() {
 					Roles: []FoundationDBStatusProcessRoleInfo{
 						{Role: string(ProcessRoleCoordinator)},
 						{
-							Role: string(ProcessClassStorage),
-							ID:   "389c23d59a646e52",
+							Role:                  string(ProcessClassStorage),
+							ID:                    "389c23d59a646e52",
+							DataLag:               FoundationDBStatusLagInfo{Seconds: 2.1227, Versions: 2122697},
+							KVStoreUsedBytes:      pointer.Int64(104878232),
+							KVStoreTotalBytes:     pointer.Int64(135012552704),
+							KVStoreFreeBytes:      pointer.Int64(84178239488),
+							KVStoreAvailableBytes: pointer.Int64(84178239488),
+							ReadLatencyStatistics: FoundationDBStatusPerfStatistics{Count: pointer.Int(334), Median: pointer.Float64(0.000102282), P99: pointer.Float64(0.000386477)},
 						},
 						{
 							Role: string(ProcessRoleResolver),
@@ -131,10 +156,24 @@ var _ = Describe("FoundationDBStatus", func() {
 						{
 							Role: string(ProcessRoleCommitProxy),
 							ID:   "0eb90e4a0ece85b3",
+							CommitLatencyStatistics: FoundationDBStatusPerfStatistics{
+								Count:  pointer.Int(31),
+								Median: pointer.Float64(0.00794888),
+								P99:    pointer.Float64(0.09054090000000001),
+							},
 						},
 						{
 							Role: string(ProcessRoleStorage),
 							ID:   "b5e42e100018bf11",
+							DataLag: FoundationDBStatusLagInfo{
+								Seconds:  0.19625800000000002,
+								Versions: 196258,
+							},
+							KVStoreUsedBytes:      pointer.Int64(104861752),
+							KVStoreTotalBytes:     pointer.Int64(135012552704),
+							KVStoreFreeBytes:      pointer.Int64(84178112512),
+							KVStoreAvailableBytes: pointer.Int64(84178112512),
+							ReadLatencyStatistics: FoundationDBStatusPerfStatistics{Count: pointer.Int(0), Median: pointer.Float64(0), P99: pointer.Float64(0)},
 						},
 					},
 					Messages: []FoundationDBStatusProcessMessage{},
@@ -213,8 +252,12 @@ var _ = Describe("FoundationDBStatus", func() {
 					UptimeSeconds: 85.0029,
 					Roles: []FoundationDBStatusProcessRoleInfo{
 						{
-							Role: string(ProcessRoleLog),
-							ID:   "2c66a861b33b2697",
+							Role:                  string(ProcessRoleLog),
+							ID:                    "2c66a861b33b2697",
+							KVStoreUsedBytes:      pointer.Int64(104861752),
+							KVStoreTotalBytes:     pointer.Int64(135012552704),
+							KVStoreFreeBytes:      pointer.Int64(84178214912),
+							KVStoreAvailableBytes: pointer.Int64(84178214912),
 						},
 					},
 					Messages: []FoundationDBStatusProcessMessage{},
@@ -237,8 +280,12 @@ var _ = Describe("FoundationDBStatus", func() {
 					UptimeSeconds: 85.003,
 					Roles: []FoundationDBStatusProcessRoleInfo{
 						{
-							Role: string(ProcessRoleLog),
-							ID:   "56cf105980ec2b07",
+							Role:                  string(ProcessRoleLog),
+							ID:                    "56cf105980ec2b07",
+							KVStoreUsedBytes:      pointer.Int64(104861752),
+							KVStoreTotalBytes:     pointer.Int64(135012552704),
+							KVStoreFreeBytes:      pointer.Int64(84178214912),
+							KVStoreAvailableBytes: pointer.Int64(84178214912),
 						},
 					},
 					Messages: []FoundationDBStatusProcessMessage{},
@@ -261,8 +308,12 @@ var _ = Describe("FoundationDBStatus", func() {
 					UptimeSeconds: 85.0027,
 					Roles: []FoundationDBStatusProcessRoleInfo{
 						{
-							Role: string(ProcessRoleLog),
-							ID:   "31754d1d7d8d6f05",
+							Role:                  string(ProcessRoleLog),
+							ID:                    "31754d1d7d8d6f05",
+							KVStoreUsedBytes:      pointer.Int64(104861752),
+							KVStoreTotalBytes:     pointer.Int64(135012552704),
+							KVStoreFreeBytes:      pointer.Int64(84178202624),
+							KVStoreAvailableBytes: pointer.Int64(84178202624),
 						},
 					},
 					Messages: []FoundationDBStatusProcessMessage{},
@@ -274,10 +325,12 @@ var _ = Describe("FoundationDBStatus", func() {
 				State:      FoundationDBStatusDataState{Description: "", Healthy: true, Name: "healthy", MinReplicasRemaining: 2},
 				TeamTrackers: []FoundationDBStatusTeamTracker{
 					{
-						Primary: true,
-						State:   FoundationDBStatusDataState{Description: "", Healthy: true, Name: "healthy", MinReplicasRemaining: 2},
+						Primary:          true,
+						State:            FoundationDBStatusDataState{Description: "", Healthy: true, Name: "healthy", MinReplicasRemaining: 2},
+						UnhealthyServers: pointer.Int64(0),
 					},
 				},
+				TotalDiskUsedBytes: pointer.Int64(629203472),
 			},
 			FullReplication: true,
 			Clients: FoundationDBStatusClusterClientInfo{
@@ -399,6 +452,7 @@ var _ = Describe("FoundationDBStatus", func() {
 				},
 			},
 			Layers: FoundationDBStatusLayerInfo{
+				Valid:  pointer.Bool(true),
 				Backup: FoundationDBStatusBackupInfo{Paused: false, Tags: nil},
 				Error:  "",
 			},
@@ -442,6 +496,11 @@ var _ = Describe("FoundationDBStatus", func() {
 			BounceImpact: FoundationDBBounceImpact{
 				CanCleanBounce: pointer.Bool(true),
 			},
+			DatabaseAvailable: pointer.Bool(true),
+			ActivePrimaryDC:   pointer.String(""),
+			DatabaseLockState: FoundationDBStatusLockState{
+				Locked: pointer.Bool(false),
+			},
 		}
 
 		It("should parse all values correctly", func() {
@@ -452,7 +511,28 @@ var _ = Describe("FoundationDBStatus", func() {
 			statusParsed := FoundationDBStatus{}
 			err = statusDecoder.Decode(&statusParsed)
 			Expect(err).NotTo(HaveOccurred())
-			Expect(statusParsed.Cluster).To(Equal(status))
+
+			// Gomega needs some prodding to effectively format diffs between these status objects, so we compare
+			// the fields one by one.
+			clusterInfoParsed := statusParsed.Cluster
+			Expect(clusterInfoParsed.DatabaseConfiguration).To(Equal(clusterInfo.DatabaseConfiguration))
+			Expect(clusterInfoParsed.Processes).To(ConsistOf(slices.Collect(maps.Values(clusterInfo.Processes))))
+			Expect(clusterInfoParsed.Data).To(Equal(clusterInfo.Data))
+			Expect(clusterInfoParsed.FullReplication).To(Equal(clusterInfo.FullReplication))
+			Expect(clusterInfoParsed.Generation).To(Equal(clusterInfo.Generation))
+			Expect(clusterInfoParsed.MaintenanceZone).To(Equal(clusterInfo.MaintenanceZone))
+			Expect(clusterInfoParsed.Clients.Count).To(Equal(clusterInfo.Clients.Count))
+			Expect(clusterInfoParsed.Clients.SupportedVersions).To(HaveExactElements(clusterInfoParsed.Clients.SupportedVersions))
+			Expect(clusterInfoParsed.Layers).To(Equal(clusterInfo.Layers))
+			Expect(clusterInfoParsed.Logs).To(HaveExactElements(clusterInfo.Logs))
+			Expect(clusterInfoParsed.Qos).To(Equal(clusterInfo.Qos))
+			Expect(clusterInfoParsed.FaultTolerance).To(Equal(clusterInfo.FaultTolerance))
+			Expect(clusterInfoParsed.MaintenanceZone).To(Equal(clusterInfo.MaintenanceZone))
+			Expect(clusterInfoParsed.IncompatibleConnections).To(HaveExactElements(clusterInfo.IncompatibleConnections))
+			Expect(clusterInfoParsed.BounceImpact).To(Equal(clusterInfo.BounceImpact))
+			Expect(clusterInfoParsed.DatabaseAvailable).To(Equal(clusterInfo.DatabaseAvailable))
+			Expect(clusterInfoParsed.ActivePrimaryDC).To(Equal(clusterInfo.ActivePrimaryDC))
+			Expect(clusterInfoParsed.DatabaseLockState).To(Equal(clusterInfo.DatabaseLockState))
 		})
 	})