Adding multigrescluster and tablegroup controller

.github/workflows/_reusable-test-coverage.yaml

@@ -44,7 +44,7 @@ jobs:
           fetch-depth: 0 # Fetch all history for changed-modules detection
 
       - name: Fetch base branch for comparison
-         run: |
+        run: |
           TARGET_BRANCH="${{ github.base_ref || 'main' }}"
           echo "Fetching comparison target: $TARGET_BRANCH"
           git fetch origin ${TARGET_BRANCH}:refs/remotes/origin/${TARGET_BRANCH}
@@ -93,6 +93,11 @@ jobs:
           echo "Previous test coverage: Not found"
           fi
 
+          ## Setup Workspace for Tooling
+          # This forces Go to look at local directories for modules instead of trying to fetch them from online.
+          go work init
+          go work use -r .
+
           ## Step 2. Check new coverage
           # Get into directory so that go tool cover can work
           go tool cover -func=coverage/combined.out > /tmp/coverage.txt
@@ -108,15 +113,16 @@ jobs:
       # change is made, the previous test reports will be minimised, leaving the
       # only relevant comment.
       - name: Check and report
-        uses: actions/github-script@v6 # Based on Node.js v16
-        if: always() &&
-          github.event_name == 'pull_request'
+        uses: actions/github-script@v6
+        if: always() && github.event_name == 'pull_request'
+        env:
+          COVERAGE_THRESHOLD: ${{ inputs.COVERAGE_THRESHOLD }}
         with:
           retries: 3
           script: |
             const fs = require('fs/promises')
 
-            // 1. Retrieve existing bot comments for the PR
+            // 1. Retrieve existing bot comments
             const { data: comments } = await github.rest.issues.listComments({
               ...context.repo,
               issue_number: context.issue.number,
@@ -126,23 +132,44 @@ jobs:
                 comment.body.includes('Go Test Coverage Report')
             });
 
-            // 2. Prepare comment
-            const report = await fs.readFile('/tmp/coverage.txt')
-            const overallStatus =
-              ${{ inputs.COVERAGE_THRESHOLD }} > ${{ env.NEW_COVERAGE }} ?
-                "❌ FAIL: Coverage less than threshold of `${{ inputs.COVERAGE_THRESHOLD }}`" :
-              ${{ env.PREV_COVERAGE || '0' }} > ${{ env.NEW_COVERAGE }} ?
-                "❌ FAIL: Coverage less than the previous run" :
-                  "✅ PASS"
+            // 2. Safe Value Parsing
+            const threshold = parseFloat(process.env.COVERAGE_THRESHOLD) || 0;
+            const newCovStr = process.env.NEW_COVERAGE;
+            const prevCovStr = process.env.PREV_COVERAGE;
+
+            const newCov = parseFloat(newCovStr); // NaN if undefined/empty
+            const prevCov = parseFloat(prevCovStr) || 0;
+
+            let overallStatus = "⚠️ SKIPPED (Build Failed?)";
+
+            // Only calculate status if we actually have coverage data
+            if (!isNaN(newCov)) {
+                if (threshold > newCov) {
+                    overallStatus = `❌ FAIL: Coverage (${newCov}%) < Threshold (${threshold}%)`;
+                } else if (prevCov > newCov) {
+                    overallStatus = `❌ FAIL: Coverage decreased (${prevCov}% -> ${newCov}%)`;
+                } else {
+                    overallStatus = "✅ PASS";
+                }
+            }
+
+            // 3. Read Report or Default Message
+            let reportContent = "No coverage report generated (Build likely failed).";
+            try {
+                reportContent = await fs.readFile('/tmp/coverage.txt', 'utf8');
+            } catch (e) {
+                console.log("Could not read coverage file:", e.message);
+            }
+
             const comment = `### 🔬 Go Test Coverage Report
 
             #### Summary
 
             | Coverage Type          | Result                                 |
             | ---------------------- | -------------------------------------- |
-            | Threshold              | ${{ inputs.COVERAGE_THRESHOLD }}%      |
-            | Previous Test Coverage | ${{ env.PREV_COVERAGE || 'Unknown' }}% |
-            | New Test Coverage      | ${{ env.NEW_COVERAGE }}%               |
+            | Threshold              | ${threshold}%                          |
+            | Previous Test Coverage | ${prevCovStr || 'Unknown'}%            |
+            | New Test Coverage      | ${newCovStr || 'Unknown'}%             |
 
             #### Status
 
@@ -153,14 +180,14 @@ jobs:
             <details><summary>Show New Coverage</summary>
 
             \`\`\`
-            ${report}\`\`\`
+            ${reportContent}
+            \`\`\`
 
             </details>
             `;
 
-            // 3. If there are any old comments, minimize all of them first.
+            // 4. Minimize old comments
             for (const botComment of botComments) {
-              core.notice("There was an old comment found in the PR, minimizing it.")
               const query = `mutation {
                 minimizeComment(input: {classifier: OUTDATED, subjectId: "${botComment.node_id}"}) {
                   clientMutationId
@@ -169,7 +196,7 @@ jobs:
               await github.graphql(query)
             }
 
-            // 4. Create a comment with the coverage report
+            // 5. Create new comment
             github.rest.issues.createComment({
               ...context.repo,
               issue_number: context.issue.number,
@@ -178,7 +205,7 @@ jobs:
 
       # Exit with non-zero value if the test coverage has decreased or not
       # reached the threshold.
-- name: Check coverage status
+      - name: Check coverage status
         run: |
           echo "Coverage Threshold:     ${{ inputs.COVERAGE_THRESHOLD }}%"
           echo "Previous test coverage: ${{ env.PREV_COVERAGE || 'Unknown' }}%"
@@ -226,5 +253,4 @@ jobs:
           path: |
             ~/.cache/coverage.txt
             ~/.cache/go-build
-            ~/go/pkg/mod
-
+            ~/go/pkg/mod

.gitignore

@@ -20,3 +20,7 @@ bin/
 
 # kind local development
 kubeconfig.yaml
+
+
+# MacOS
+.DS_Store   

api/v1alpha1/cell_types.go

@@ -57,11 +57,11 @@ type CellSpec struct {
 
 	// TopoServer defines the local topology config.
 	// +optional
-	TopoServer LocalTopoServerSpec `json:"topoServer,omitempty"`
+	TopoServer *LocalTopoServerSpec `json:"topoServer,omitempty"`
 
 	// AllCells list for discovery.
 	// +optional
-	// +kubebuilder:validation:MaxItems=100
+	// +kubebuilder:validation:MaxItems=50
 	AllCells []CellName `json:"allCells,omitempty"`
 
 	// TopologyReconciliation flags.
@@ -71,7 +71,10 @@ type CellSpec struct {
 
 // TopologyReconciliation defines flags for the cell controller.
 type TopologyReconciliation struct {
+	// RegisterCell indicates if the cell should register itself in the topology.
 	RegisterCell bool `json:"registerCell"`
+
+	// PrunePoolers indicates if unused poolers should be removed.
 	PrunePoolers bool `json:"prunePoolers"`
 }
 
@@ -85,8 +88,13 @@ type CellStatus struct {
 	// +optional
 	Conditions []metav1.Condition `json:"conditions,omitempty"`
 
-	GatewayReplicas      int32 `json:"gatewayReplicas"`
+	// GatewayReplicas is the total number of gateway pods.
+	GatewayReplicas int32 `json:"gatewayReplicas"`
+
+	// GatewayReadyReplicas is the number of gateway pods that are ready.
 	GatewayReadyReplicas int32 `json:"gatewayReadyReplicas"`
+
+	// GatewayServiceName is the DNS name of the gateway service.
 	// +kubebuilder:validation:MaxLength=253
 	GatewayServiceName string `json:"gatewayServiceName,omitempty"`
 }

api/v1alpha1/common_types.go

@@ -45,13 +45,13 @@ type StatelessSpec struct {
 
 	// PodAnnotations are annotations to add to the pods.
 	// +optional
-	// +kubebuilder:validation:MaxProperties=64
+	// +kubebuilder:validation:MaxProperties=20
 	// +kubebuilder:validation:XValidation:rule="self.all(k, size(k) < 64 && size(self[k]) < 256)",message="annotation keys must be <64 chars and values <256 chars"
 	PodAnnotations map[string]string `json:"podAnnotations,omitempty"`
 
 	// PodLabels are additional labels to add to the pods.
 	// +optional
-	// +kubebuilder:validation:MaxProperties=64
+	// +kubebuilder:validation:MaxProperties=20
 	// +kubebuilder:validation:XValidation:rule="self.all(k, size(k) < 64 && size(self[k]) < 64)",message="label keys and values must be <64 chars"
 	PodLabels map[string]string `json:"podLabels,omitempty"`
 }

api/v1alpha1/multigrescluster_types.go

@@ -48,15 +48,15 @@ type MultigresClusterSpec struct {
 	// +optional
 	// +listType=map
 	// +listMapKey=name
-	// +kubebuilder:validation:MaxItems=100
+	// +kubebuilder:validation:MaxItems=50
 	Cells []CellConfig `json:"cells,omitempty"`
 
 	// Databases defines the logical databases, table groups, and sharding.
 	// +optional
 	// +listType=map
 	// +listMapKey=name
 	// +kubebuilder:validation:XValidation:rule="self.filter(x, has(x.default) && x.default).size() <= 1",message="only one database can be marked as default"
-	// +kubebuilder:validation:MaxItems=500
+	// +kubebuilder:validation:MaxItems=50
 	Databases []DatabaseConfig `json:"databases,omitempty"`
 }
 
@@ -218,7 +218,7 @@ type DatabaseConfig struct {
 	// +listType=map
 	// +listMapKey=name
 	// +kubebuilder:validation:XValidation:rule="self.filter(x, has(x.default) && x.default).size() <= 1",message="only one tablegroup can be marked as default"
-	// +kubebuilder:validation:MaxItems=100
+	// +kubebuilder:validation:MaxItems=20
 	TableGroups []TableGroupConfig `json:"tablegroups,omitempty"`
 }
 
@@ -237,7 +237,7 @@ type TableGroupConfig struct {
 	// +optional
 	// +listType=map
 	// +listMapKey=name
-	// +kubebuilder:validation:MaxItems=128
+	// +kubebuilder:validation:MaxItems=32
 	Shards []ShardConfig `json:"shards,omitempty"`
 }
 
@@ -272,7 +272,7 @@ type ShardOverrides struct {
 
 	// Pools overrides. Keyed by pool name.
 	// +optional
-	// +kubebuilder:validation:MaxProperties=32
+	// +kubebuilder:validation:MaxProperties=8
 	// +kubebuilder:validation:XValidation:rule="self.all(key, size(key) < 63)",message="pool names must be < 63 chars"
 	Pools map[string]PoolSpec `json:"pools,omitempty"`
 }
@@ -285,7 +285,7 @@ type ShardInlineSpec struct {
 
 	// Pools configuration. Keyed by pool name.
 	// +optional
-	// +kubebuilder:validation:MaxProperties=32
+	// +kubebuilder:validation:MaxProperties=8
 	// +kubebuilder:validation:XValidation:rule="self.all(key, size(key) < 63)",message="pool names must be < 63 chars"
 	Pools map[string]PoolSpec `json:"pools,omitempty"`
 }
@@ -305,13 +305,13 @@ type MultigresClusterStatus struct {
 	Conditions []metav1.Condition `json:"conditions,omitempty"`
 	// Cells status summary.
 	// +optional
-	// +kubebuilder:validation:MaxProperties=100
+	// +kubebuilder:validation:MaxProperties=50
 	// +kubebuilder:validation:XValidation:rule="self.all(key, size(key) < 63)",message="cell names must be < 63 chars"
 	Cells map[string]CellStatusSummary `json:"cells,omitempty"`
 
 	// Databases status summary.
 	// +optional
-	// +kubebuilder:validation:MaxProperties=500
+	// +kubebuilder:validation:MaxProperties=50
 	// +kubebuilder:validation:XValidation:rule="self.all(key, size(key) < 63)",message="database names must be < 63 chars"
 	Databases map[string]DatabaseStatusSummary `json:"databases,omitempty"`
 }

api/v1alpha1/shard_types.go

@@ -36,7 +36,7 @@ type MultiOrchSpec struct {
 	// Cells defines the list of cells where this MultiOrch should be deployed.
 	// If empty, it defaults to all cells where pools are defined.
 	// +optional
-	// +kubebuilder:validation:MaxItems=100
+	// +kubebuilder:validation:MaxItems=50
 	Cells []CellName `json:"cells,omitempty"`
 }
 
@@ -49,7 +49,7 @@ type PoolSpec struct {
 
 	// Cells defines the list of cells where this Pool should be deployed.
 	// +optional
-	// +kubebuilder:validation:MaxItems=100
+	// +kubebuilder:validation:MaxItems=50
 	Cells []CellName `json:"cells,omitempty"`
 
 	// ReplicasPerCell is the desired number of pods PER CELL in this pool.
@@ -83,34 +83,44 @@ type PoolSpec struct {
 
 // ShardSpec defines the desired state of Shard.
 type ShardSpec struct {
+	// DatabaseName is the name of the logical database this shard belongs to.
 	// +kubebuilder:validation:MaxLength=63
 	DatabaseName string `json:"databaseName"`
+
+	// TableGroupName is the name of the table group this shard belongs to.
 	// +kubebuilder:validation:MaxLength=63
 	TableGroupName string `json:"tableGroupName"`
+
+	// ShardName is the specific identifier for this shard (e.g. "0").
 	// +kubebuilder:validation:MaxLength=63
 	ShardName string `json:"shardName"`
 
-	// Images required.
+	// Images defines the container images to be used by this shard (defined globally at MultigresCluster).
 	Images ShardImages `json:"images"`
 
-	// GlobalTopoServer reference.
+	// GlobalTopoServer is a reference to the global topology server.
 	GlobalTopoServer GlobalTopoServerRef `json:"globalTopoServer"`
 
-	// MultiOrch fully resolved spec.
+	// MultiOrch is the fully resolved configuration for the shard orchestrator.
 	MultiOrch MultiOrchSpec `json:"multiorch"`
 
-	// Pools fully resolved spec.
-	// +kubebuilder:validation:MaxProperties=32
+	// Pools is the map of fully resolved data pool configurations.
+	// +kubebuilder:validation:MaxProperties=8
 	// +kubebuilder:validation:XValidation:rule="self.all(key, size(key) < 63)",message="pool names must be < 63 chars"
 	Pools map[string]PoolSpec `json:"pools"`
 }
 
 // ShardImages defines the images required for a Shard.
 type ShardImages struct {
+	// MultiOrch is the image for the shard orchestrator.
 	// +kubebuilder:validation:MaxLength=512
 	MultiOrch string `json:"multiorch"`
+
+	// MultiPooler is the image for the connection pooler sidecar.
 	// +kubebuilder:validation:MaxLength=512
 	MultiPooler string `json:"multipooler"`
+
+	// Postgres is the image for the postgres database.
 	// +kubebuilder:validation:MaxLength=512
 	Postgres string `json:"postgres"`
 }
@@ -127,10 +137,13 @@ type ShardStatus struct {
 
 	// Cells is a list of cells this shard is currently deployed to.
 	// +optional
-	// +kubebuilder:validation:MaxItems=100
+	// +kubebuilder:validation:MaxItems=50
 	Cells []CellName `json:"cells,omitempty"`
 
-	OrchReady  bool `json:"orchReady"`
+	// OrchReady indicates if the MultiOrch component is ready.
+	OrchReady bool `json:"orchReady"`
+
+	// PoolsReady indicates if all data pools are ready.
 	PoolsReady bool `json:"poolsReady"`
 }
 

api/v1alpha1/shardtemplate_types.go

@@ -31,7 +31,7 @@ type ShardTemplateSpec struct {
 	MultiOrch *MultiOrchSpec `json:"multiorch,omitempty"`
 
 	// +optional
-	// +kubebuilder:validation:MaxProperties=32
+	// +kubebuilder:validation:MaxProperties=8
 	// +kubebuilder:validation:XValidation:rule="self.all(key, size(key) < 63)",message="pool names must be < 63 chars"
 	Pools map[string]PoolSpec `json:"pools,omitempty"`
 }