fix: loadbalancing with nginx ingress controller

Signed-off-by: 90DY <forward@90dy.ltd>

Author: 90dy

.github/workflows/gitops.yml .github/workflows/apply.yml.github/workflows/gitops.yml renamed to .github/workflows/apply.yml

@@ -40,6 +40,9 @@ jobs:
         # Hide logs for public repositories
         run: make apply 1>/dev/null 2>/dev/null 
 
+      - name: Tests changes
+        run: make test
+
       - name: Bump version and push tag
         id: tag_version
         uses: mathieudutour/github-tag-action@v6.2

Makefile

@@ -3,6 +3,8 @@ get-secret = $(shell read -s -p "$(1): " secret; echo $$secret; echo 1>&0)
 
 docker-run := docker run $(shell [ -t 0 ] && echo -it || echo -i) -e DOMAIN_NAME=${DOMAIN_NAME}
 
+export DOMAIN_NAME ?= ${domain}
+
 .PHONY: build
 build:
 	docker build -t kubespray -f Dockerfile .
@@ -42,16 +44,22 @@ ${HOME}/.cntb.yaml:
 .PHONY: generate
 generate: ## Generate the ansible inventory
 generate: .cntb .cntb/private-networks.json .cntb/instances.json
-	@deno -A ./templates/_generate.ts
+	@deno -A ./generate.ts
 .cntb:
 	@mkdir -p .cntb
-.cntb/private-networks.json: .FORCE
+.cntb/private-networks.json:
 	@cntb get privateNetworks --output json > .cntb/private-networks.json
-.cntb/instances.json: .FORCE
+.cntb/instances.json:
 	@cntb get instances --output json > .cntb/instances.json
 
 .PHONY: list-nodes
 list-nodes: ## List all nodes in the cluster
 list-nodes: generate
 	@echo "Nodes:"
-	@cat .cntb/instances.json | jq -r '.[] | .displayName' | grep '^${DOMAIN_NAME}-'
+	@cat .cntb/instances.json | jq -r '.[] | .displayName' | grep '^${DOMAIN_NAME}-'
+
+.PHONY: test
+test: ## Test basic cluster functionalities
+test: generate
+	@echo "Test basic cluster functionalities..."
+	@deno test -A

README.md

@@ -57,7 +57,7 @@ make apply
 
 ## Configuration & Node Naming
 
-- **Core Files**: `templates/inventory.ini.ts`, `templates/group_vars/k8s_cluster/k8s-cluster.yml.ts`, `templates/group_vars/k8s_cluster/addons.yml.ts`
+- **Core Files**: `inventory.ini.ts`, `kubeconfig.yml.ts`, `group_vars/k8s_cluster/k8s-cluster.yml.ts`, `group_vars/k8s_cluster/addons.yml.ts`
 - **Node Naming**: Each node name must start with the domain name prefix (`ctnr.io-`) followed by the role:
   - Control Plane: `<domain-name>-control-plane-X[-etcd][-worker]`
   - ETCD: `<domain-name>-etcd-X`
@@ -74,9 +74,19 @@ The first control plane node is critical for cluster stability. Modifying or rem
 2. Apply the configuration
 3. Only then remove the original control plane node
 
+## Testing
+
+The project includes automated tests to verify that key components are working correctly:
+
+- **Ingress Testing**: Tests that the Nginx ingress controller is properly configured and can route traffic to services
+  - Run with: `make test-ingress` or `deno test -A tests/ingress_test.ts`
+  - The test creates a test namespace, deployment, service, and ingress with a custom domain name
+  - It verifies connectivity using host-based routing with the domain name from your configuration
+  - Test fixtures are located in `tests/fixtures/` directory
+
 ## Next Steps & Roadmap
 
-- **Next**: Validate deployment, implement volume provisioning, configure auto-scaling, deploy workloads, add monitoring
+- **Next**: Implement volume provisioning, configure auto-scaling, deploy workloads, add monitoring
 - **Roadmap**: Automated provisioning, one-click deployment, backup solution, multi-region support, performance benchmarks
 
 ## Professional Support
@@ -94,3 +104,4 @@ Email: contact@ctnr.io
 ## License
 
 MIT License - see [LICENSE](LICENSE) file
+

generate.ts

@@ -0,0 +1,23 @@
+#!/usr/bin/env -S deno run -A
+
+import "npm:dotenv/config";
+import fs from "node:fs/promises";
+import * as path from "jsr:@std/path";
+
+const rootDir = path.dirname(path.fromFileUrl(import.meta.url));
+
+const templateFiles = await fs
+  .readdir(rootDir, { withFileTypes: true, recursive: true, encoding: "utf-8" })
+  .then((files) =>
+    files.filter((file) => file.isFile() && ["ini", "yml", "yaml", "json"].some((ext) => file.name.endsWith(`.${ext}.ts`)))
+  );
+
+await Promise.all(
+  templateFiles.map(async (file) => {
+    const templatePath = `${file.parentPath}/${file.name}`;
+    const outputFilePath = `${file.parentPath}/${file.name.replace(/.ts$/, "")}`;
+    const { default: content } = await import(templatePath);
+    console.info("Generating", outputFilePath, "...");
+    await fs.writeFile(outputFilePath, content);
+  })
+);

templates/group_vars/all/all.yml.ts group_vars/all/all.yml.tstemplates/group_vars/all/all.yml.ts renamed to group_vars/all/all.yml.ts

@@ -1,4 +1,4 @@
-import { apiserverPort } from "../../_helpers.ts";
+import { apiserverPort } from "../../helpers.ts";
 
 const yaml = String.raw
 

…tes/group_vars/k8s_cluster/addons.yml.ts group_vars/k8s_cluster/addons.yml.tstemplates/group_vars/k8s_cluster/addons.yml.ts renamed to group_vars/k8s_cluster/addons.yml.ts templates/group_vars/k8s_cluster/addons.yml.ts renamed to group_vars/k8s_cluster/addons.yml.ts

@@ -1,10 +1,10 @@
-import { apiserverPrivateIp, apiserverPublicIp, nodes } from "../../_helpers.ts";
-import { privateNetworks } from "../../_helpers.ts";
+import { apiserverPrivateIp, workers } from "../../helpers.ts";
+import { privateNetworks } from "../../helpers.ts";
 
 const yaml = String.raw;
 
-const privateIpRanges = "[" + privateNetworks.map((pn) => pn.cidr).join(", ") + "]";
-const publicIpRanges = "[" + nodes.map((node) => `${node.publicIp}/32`).join(", ") + "]";
+const internalIpRanges = "[" + privateNetworks.map((pn) => pn.cidr).join(", ") + "]";
+const externalIpRanges = "[" + workers.map((node) => `${node.publicIp}/32`).join(", ") + "]";
 
 export default yaml`
 ---
@@ -16,13 +16,13 @@ export default yaml`
 helm_enabled: false
 
 # Registry deployment
-registry_enabled: false
+registry_enabled: false 
 # registry_namespace: kube-system
 # registry_storage_class: ""
 # registry_disk_size: "10Gi"
 
 # Metrics Server deployment
-metrics_server_enabled: false
+metrics_server_enabled: true 
 # metrics_server_container_port: 10250
 # metrics_server_kubelet_insecure_tls: true
 # metrics_server_metric_resolution: 15s
@@ -31,7 +31,7 @@ metrics_server_enabled: false
 # metrics_server_replicas: 1
 
 # Rancher Local Path Provisioner
-local_path_provisioner_enabled: false
+local_path_provisioner_enabled: true 
 # local_path_provisioner_namespace: "local-path-storage"
 # local_path_provisioner_storage_class: "local-path"
 # local_path_provisioner_reclaim_policy: Delete
@@ -43,7 +43,7 @@ local_path_provisioner_enabled: false
 # local_path_provisioner_helper_image_tag: "latest"
 
 # Local volume provisioner deployment
-local_volume_provisioner_enabled: false
+local_volume_provisioner_enabled: true 
 # local_volume_provisioner_namespace: kube-system
 # local_volume_provisioner_nodelabels:
 #   - kubernetes.io/hostname
@@ -110,21 +110,23 @@ gateway_api_enabled: true
 # gateway_api_experimental_channel: false
 
 # Nginx ingress controller deployment
-ingress_nginx_enabled: false
-# ingress_nginx_host_network: false
-# ingress_nginx_service_type: LoadBalancer
-# ingress_nginx_service_annotations:
-#   example.io/loadbalancerIPs: 1.2.3.4
+ingress_nginx_enabled: true 
+ingress_nginx_host_network: false
+ingress_nginx_service_type: LoadBalancer
+ingress_nginx_service_annotations:
+  metallb.universe.tf/address-pool: external 
+  # example.io/loadbalancerIPs: 1.2.3.4
 # ingress_nginx_service_nodeport_http: 30080
 # ingress_nginx_service_nodeport_https: 30081
 ingress_publish_status_address: ""
 # ingress_nginx_nodeselector:
 #   kubernetes.io/os: "linux"
-# ingress_nginx_tolerations:
-#   - key: "node-role.kubernetes.io/control-plane"
-#     operator: "Equal"
-#     value: ""
-#     effect: "NoSchedule"
+# Permit ingress traffic from control plane nodes
+ingress_nginx_tolerations:
+  - key: "node-role.kubernetes.io/control-plane"
+    operator: "Equal"
+    value: ""
+    effect: "NoSchedule"
 # ingress_nginx_namespace: "ingress-nginx"
 # ingress_nginx_insecure_port: 80
 # ingress_nginx_secure_port: 443
@@ -217,16 +219,16 @@ metallb_config:
   # To use specific pool for services, you can annotate the service with the following annotation:
   # metallb.universe.tf/address-pool
   address_pools:
-    public:
-      ip_range: ${publicIpRanges}
+    internal:
+      ip_range: ${internalIpRanges}
       auto_assign: true
-    private:
-      ip_range: ${privateIpRanges}
+    external:
+      ip_range: ${externalIpRanges}
       auto_assign: true
-  # Contabo cannot use BGP, so we use Layer 2 mode
   layer2:
-    - public
-    - private 
+    # internal ip is for all services, external ip is for ingresses like nginx ingress controller
+    - internal 
+    - external 
   # layer3:
   #   defaults:
   #     peer_port: 179
@@ -270,11 +272,11 @@ krew_enabled: false
 krew_root_dir: "/usr/local/krew"
 
 # Kube VIP
-kube_vip_enabled: true
-kube_vip_arp_enabled: true 
-kube_vip_lb_enable: true
+kube_vip_enabled: true 
+kube_vip_arp_enabled: true
+kube_vip_lb_enable: false 
 kube_vip_controlplane_enabled: true
-kube_vip_address: ${apiserverPublicIp}  # This becomes the VIP
+kube_vip_address: ${apiserverPrivateIp}  # This becomes the VIP
 loadbalancer_apiserver:
   address: "{{ kube_vip_address }}"
   port: 6443
@@ -283,10 +285,10 @@ loadbalancer_apiserver:
 # kube_vip_dns_mode: first
 # kube_vip_cp_detect: false
 # kube_vip_leasename: plndr-cp-lock
-# kube_vip_enable_node_labeling: false
+kube_vip_enable_node_labeling: true 
 
 # Node Feature Discovery
-node_feature_discovery_enabled: false
+node_feature_discovery_enabled: true 
 # node_feature_discovery_gc_sa_name: node-feature-discovery
 # node_feature_discovery_gc_sa_create: false
 # node_feature_discovery_worker_sa_name: node-feature-discovery

…roup_vars/k8s_cluster/k8s-cluster.yml.ts group_vars/k8s_cluster/k8s-cluster.yml.tstemplates/group_vars/k8s_cluster/k8s-cluster.yml.ts renamed to group_vars/k8s_cluster/k8s-cluster.yml.ts templates/group_vars/k8s_cluster/k8s-cluster.yml.ts renamed to group_vars/k8s_cluster/k8s-cluster.yml.ts

@@ -1,8 +1,8 @@
-import { controlPlanes } from "../../_helpers.ts";
+import { controlPlanes, domainName } from "../../helpers.ts";
 
 const yaml = String.raw;
 
-const supplementary_addresses_in_ssl_keys = `[${[...new Set(controlPlanes.map((node) => [node.publicIp, node.privateIp]).flat())].join(", ")}]`;
+const supplementary_addresses_in_ssl_keys = `[${domainName}, ${[...new Set(controlPlanes.map((node) => [node.publicIp, node.privateIp]).flat())].join(", ")}]`;
 
 export default yaml`
 # This configuration file is auto-generated by 'make generate'

templates/_helpers.ts helpers.tstemplates/_helpers.ts renamed to helpers.ts

@@ -1,7 +1,7 @@
 import * as process from "node:process";
 
-const { default: allInstances } = await import("../.cntb/instances.json", { with: { type: "json" } });
-const { default: allPrivateNetworks } = await import("../.cntb/private-networks.json", { with: { type: "json" } });
+const { default: allInstances } = await import("./.cntb/instances.json", { with: { type: "json" } });
+const { default: allPrivateNetworks } = await import("./.cntb/private-networks.json", { with: { type: "json" } });
 
 export const domainName = process.env.DOMAIN_NAME;
 if (!domainName) {

templates/inventory.ini.ts inventory.ini.tstemplates/inventory.ini.ts renamed to inventory.ini.ts

@@ -1,4 +1,4 @@
-import { controlPlanes, etcds, Node, workers } from "./_helpers.ts";
+import { controlPlanes, etcds, Node, workers } from "./helpers.ts";
 
 const ini = String.raw;
 

templates/kubeconfig.yml.ts kubeconfig.yml.tstemplates/kubeconfig.yml.ts renamed to kubeconfig.yml.ts

@@ -1,4 +1,4 @@
-import { apiserverPublicIp, apiserverPort, domainName } from "./_helpers.ts";
+import { apiserverPublicIp, apiserverPort, domainName } from "./helpers.ts";
 import * as util from "node:util";
 import * as childProcess from "node:child_process";
 import { encodeBase64 } from "jsr:@std/encoding/base64";