Skip to content
This repository was archived by the owner on Oct 27, 2025. It is now read-only.

Commit 26ccb2d

Browse files
90dy90dy
committed
feat(core): first version
Signed-off-by: 90DY <forward@90dy.ltd>
1 parent 3162c66 commit 26ccb2d

File tree

26 files changed

+1898
-81
lines changed

26 files changed

+1898
-81
lines changed

‎.github/workflows/apply.yml‎

Lines changed: 9 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -19,20 +19,16 @@ jobs:
1919
DOMAIN_NAME: ${{ vars.DOMAIN_NAME }}
2020
permissions:
2121
contents: write
22-
# strategy:
23-
# matrix:
24-
# envionment:
25-
# - 0.eu.cntb.hacl.${{ vars.DOMAIN_NAME }}
26-
# - 1.eu.cntb.hacl.${{ vars.DOMAIN_NAME }}
27-
# - 2.eu.cntb.hacl.${{ vars.DOMAIN_NAME }}
28-
# max-parallel: 1
29-
# fail-fast: false
3022
steps:
3123
- name: Checkout
3224
uses: actions/checkout@v2
3325

34-
- name: Get private key from secrets
35-
run: echo "${{ secrets.PRIVATE_KEY }}" > ${{ vars.DOMAIN_NAME }}-private.key
26+
- name: Set up SSH key
27+
run: |
28+
mkdir -p ~/.ssh
29+
echo "${{ secrets.PRIVATE_KEY }}" > ~/.ssh/id_rsa
30+
chmod 600 ~/.ssh/id_rsa
31+
ssh-keygen -y -f ~/.ssh/id_rsa > ~/.ssh/id_rsa.pub
3632
3733
- name: Get Contabo YAML from secrets && install cntb
3834
run: |
@@ -44,9 +40,9 @@ jobs:
4440
with:
4541
deno-version: v2.x
4642

47-
- name: Apply changes
43+
- name: Create or update cluster
4844
# Hide logs for public repositories
49-
run: make apply # 1>/dev/null
45+
run: make create-cluster # 1>/dev/null
5046

5147
- name: Tests changes
5248
run: make test # 1>/dev/null
@@ -61,4 +57,4 @@ jobs:
6157
with:
6258
tag: ${{ steps.tag_version.outputs.new_tag }}
6359
name: Release ${{ steps.tag_version.outputs.new_tag }}
64-
body: ${{ steps.tag_version.outputs.changelog }}
60+
body: ${{ steps.tag_version.outputs.changelog }}

‎Dockerfile‎

Lines changed: 31 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,36 @@
1-
FROM quay.io/kubespray/kubespray:v2.27.0
1+
FROM ubuntu:22.04
22

3+
# Install dependencies
4+
RUN apt-get update && apt-get install -y \
5+
apt-transport-https \
6+
ca-certificates \
7+
curl \
8+
gnupg \
9+
lsb-release \
10+
openssh-client \
11+
jq \
12+
vim \
13+
&& rm -rf /var/lib/apt/lists/*
14+
15+
# Install kubectl
16+
RUN curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg --dearmor -o /usr/share/keyrings/kubernetes-archive-keyring.gpg \
17+
&& echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | tee /etc/apt/sources.list.d/kubernetes.list \
18+
&& apt-get update \
19+
&& apt-get install -y kubectl \
20+
&& rm -rf /var/lib/apt/lists/*
21+
22+
# Set environment variables
323
ENV DOMAIN_NAME=
24+
ENV KUBECONFIG=/config/kubeconfig.yml
25+
26+
# Create directories
27+
RUN mkdir -p /config
428

5-
VOLUME /inventory
29+
# Set working directory
30+
WORKDIR /config
631

7-
ENTRYPOINT [ "ansible-playbook", "--private-key=/inventory/${DOMAIN_NAME}-private.key", "-i=/inventory/inventory.ini", "--forks=50" ]
32+
# Volume for configuration files
33+
VOLUME /config
834

9-
CMD [ "cluster.yml" ]
35+
# Default command
36+
CMD ["bash"]

‎Makefile‎

Lines changed: 128 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -3,25 +3,112 @@ get-secret = $(shell read -s -p "$(1): " secret; echo $$secret; echo 1>&0)
33

44
export DOMAIN_NAME ?= ${domain}
55

6-
docker-run := docker run $(shell [ -t 0 ] && echo -it || echo -i) -e DOMAIN_NAME=${DOMAIN_NAME}
6+
SSH_OPTS := -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
77

8-
.PHONY: build
9-
build:
10-
docker build -t kubespray -f Dockerfile .
8+
# Default Kubernetes version
9+
K8S_VERSION ?= 1.31.4
1110

12-
.PHONY: apply
13-
apply: ## Apply the kubernetes cluster
14-
apply: generate ${DOMAIN_NAME}-private.key build
15-
${docker-run} -v .:/inventory kubespray cluster.yml
11+
# Default CNI
12+
CNI ?= calico
1613

17-
.PHONY: reset
18-
reset: ## Reset the kubernetes cluster
19-
reset: generate ${DOMAIN_NAME}-private.key build
20-
${docker-run} -v .:/inventory kubespray reset.yml
14+
# Default pod CIDR
15+
POD_CIDR ?= 10.244.0.0/16
2116

22-
.PHONY: ${DOMAIN_NAME}-private.key
23-
${DOMAIN_NAME}-private.key: .FORCE
24-
@chmod 400 $@
17+
# Default service CIDR
18+
SERVICE_CIDR ?= 10.96.0.0/12
19+
20+
# Default control plane endpoint
21+
CONTROL_PLANE_ENDPOINT ?= $(shell cat .cntb/instances.json | jq -r '.[] | select(.displayName | startswith("$(DOMAIN_NAME)_control-plane-0")) | .ipv4')
22+
23+
.PHONY: create-cluster
24+
create-cluster: ## Create a new Kubernetes cluster based on available Contabo nodes
25+
create-cluster: generate
26+
@echo "Creating Kubernetes cluster with kubeadm..."
27+
@echo "Initializing control plane on $(CONTROL_PLANE_ENDPOINT)..."
28+
@ssh $(SSH_OPTS) root@$(CONTROL_PLANE_ENDPOINT) "kubeadm init --kubernetes-version=$(K8S_VERSION) \
29+
--pod-network-cidr=$(POD_CIDR) \
30+
--service-cidr=$(SERVICE_CIDR) \
31+
--control-plane-endpoint=$(CONTROL_PLANE_ENDPOINT):6443 \
32+
--upload-certs"
33+
@echo "Installing CNI ($(CNI))..."
34+
@if [ "$(CNI)" = "calico" ]; then \
35+
ssh $(SSH_OPTS) root@$(CONTROL_PLANE_ENDPOINT) "kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f https://docs.projectcalico.org/manifests/calico.yaml"; \
36+
elif [ "$(CNI)" = "flannel" ]; then \
37+
ssh $(SSH_OPTS) root@$(CONTROL_PLANE_ENDPOINT) "kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f https://raw.githubusercontent.com/flannel-io/flannel/master/Documentation/kube-flannel.yml"; \
38+
fi
39+
@echo "Fetching kubeconfig..."
40+
@ssh $(SSH_OPTS) root@$(CONTROL_PLANE_ENDPOINT) "cat /etc/kubernetes/admin.conf" > kubeconfig.yml
41+
@echo "Cluster created successfully! Kubeconfig saved to kubeconfig.yml"
42+
43+
.PHONY: add-control-plane
44+
add-control-plane: ## Add a control plane node to the cluster
45+
add-control-plane: node ?= $(call get-input,Node name (e.g. $(DOMAIN_NAME)_control-plane-1))
46+
add-control-plane: generate
47+
@echo "Adding control plane node $(node)..."
48+
@NODE_IP=$(shell cat .cntb/instances.json | jq -r '.[] | select(.displayName == "$(node)") | .ipv4') && \
49+
JOIN_CMD=$$(ssh $(SSH_OPTS) root@$(CONTROL_PLANE_ENDPOINT) "kubeadm token create --print-join-command") && \
50+
CERTIFICATE_KEY=$$(ssh $(SSH_OPTS) root@$(CONTROL_PLANE_ENDPOINT) "kubeadm init phase upload-certs --upload-certs | tail -1") && \
51+
ssh $(SSH_OPTS) root@$$NODE_IP "$$JOIN_CMD --control-plane --certificate-key $$CERTIFICATE_KEY"
52+
@echo "Control plane node $(node) added successfully!"
53+
54+
.PHONY: add-worker
55+
add-worker: ## Add a worker node to the cluster
56+
add-worker: node ?= $(call get-input,Node name (e.g. $(DOMAIN_NAME)_worker-0))
57+
add-worker: generate
58+
@echo "Adding worker node $(node)..."
59+
@NODE_IP=$(shell cat .cntb/instances.json | jq -r '.[] | select(.displayName == "$(node)") | .ipv4') && \
60+
JOIN_CMD=$$(ssh $(SSH_OPTS) root@$(CONTROL_PLANE_ENDPOINT) "kubeadm token create --print-join-command") && \
61+
ssh $(SSH_OPTS) root@$$NODE_IP "$$JOIN_CMD"
62+
@echo "Worker node $(node) added successfully!"
63+
64+
.PHONY: add-etcd
65+
add-etcd: ## Add an etcd node to the cluster (requires manual configuration)
66+
add-etcd: node ?= $(call get-input,Node name (e.g. $(DOMAIN_NAME)_etcd-0))
67+
add-etcd: generate
68+
@echo "Adding etcd node $(node)..."
69+
@echo "Note: Adding external etcd nodes requires manual configuration."
70+
@echo "Please follow the kubeadm documentation for setting up an external etcd cluster."
71+
@NODE_IP=$(shell cat .cntb/instances.json | jq -r '.[] | select(.displayName == "$(node)") | .ipv4')
72+
@echo "Node IP: $$NODE_IP"
73+
74+
.PHONY: remove-node
75+
remove-node: ## Remove a node from the cluster
76+
remove-node: node ?= $(call get-input,Node name to remove)
77+
remove-node: generate
78+
@echo "Removing node $(node)..."
79+
@NODE_IP=$(shell cat .cntb/instances.json | jq -r '.[] | select(.displayName == "$(node)") | .ipv4') && \
80+
ssh $(SSH_OPTS) root@$(CONTROL_PLANE_ENDPOINT) "kubectl --kubeconfig=/etc/kubernetes/admin.conf drain $(node) --ignore-daemonsets --delete-emptydir-data" && \
81+
ssh $(SSH_OPTS) root@$(CONTROL_PLANE_ENDPOINT) "kubectl --kubeconfig=/etc/kubernetes/admin.conf delete node $(node)" && \
82+
ssh $(SSH_OPTS) root@$$NODE_IP "kubeadm reset -f"
83+
@echo "Node $(node) removed successfully!"
84+
85+
.PHONY: upgrade-cluster
86+
upgrade-cluster: ## Upgrade the Kubernetes cluster
87+
upgrade-cluster: version ?= $(call get-input,Kubernetes version to upgrade to)
88+
upgrade-cluster: generate
89+
@echo "Upgrading control plane nodes to version $(version)..."
90+
@for node in $$(cat .cntb/instances.json | jq -r '.[] | select(.displayName | contains("control-plane")) | .ipv4'); do \
91+
echo "Upgrading control plane node $$node..."; \
92+
ssh $(SSH_OPTS) root@$$node "apt-get update && apt-get install -y kubeadm=$(version)-00 && kubeadm upgrade apply $(version) -y && apt-get install -y kubelet=$(version)-00 kubectl=$(version)-00 && systemctl restart kubelet"; \
93+
done
94+
@echo "Upgrading worker nodes to version $(version)..."
95+
@for node in $$(cat .cntb/instances.json | jq -r '.[] | select(.displayName | contains("worker")) | .ipv4'); do \
96+
echo "Upgrading worker node $$node..."; \
97+
ssh $(SSH_OPTS) root@$$node "apt-get update && apt-get install -y kubeadm=$(version)-00 && kubeadm upgrade node && apt-get install -y kubelet=$(version)-00 kubectl=$(version)-00 && systemctl restart kubelet"; \
98+
done
99+
@echo "Cluster upgraded to version $(version) successfully!"
100+
101+
.PHONY: reset-cluster
102+
reset-cluster: ## Reset the entire Kubernetes cluster
103+
reset-cluster: generate
104+
@echo "Resetting the entire Kubernetes cluster..."
105+
@echo "This will remove all nodes from the cluster and reset kubeadm on each node."
106+
@read -p "Are you sure you want to continue? [y/N] " confirm && [ "$$confirm" = "y" ] || exit 1
107+
@for node in $$(cat .cntb/instances.json | jq -r '.[] | select(.displayName | startswith("$(DOMAIN_NAME)_")) | .ipv4'); do \
108+
echo "Resetting node $$node..."; \
109+
ssh $(SSH_OPTS) root@$$node "kubeadm reset -f"; \
110+
done
111+
@echo "Cluster reset successfully!"
25112

26113
.PHONY: .FORCE
27114
.FORCE:
@@ -42,21 +129,42 @@ ${HOME}/.cntb.yaml:
42129
--oauth2-password='${oauth2-password}'
43130

44131
.PHONY: generate
45-
generate: ## Generate the ansible inventory
132+
generate: ## Generate the node information
46133
generate: .cntb .cntb/private-networks.json .cntb/instances.json
47134
@deno -A ./generate.ts
48-
.cntb:
135+
.cntb: .FORCE
49136
@mkdir -p .cntb
50-
.cntb/private-networks.json:
137+
.cntb/private-networks.json: .FORCE
51138
@cntb get privateNetworks --output json > .cntb/private-networks.json
52-
.cntb/instances.json:
139+
.cntb/instances.json: .FORCE
53140
@cntb get instances --output json > .cntb/instances.json
54141

55142
.PHONY: list-nodes
56143
list-nodes: ## List all nodes in the cluster
57144
list-nodes: generate
58145
@echo "Nodes:"
59-
@cat .cntb/instances.json | jq -r '.[] | .displayName' | grep '^${DOMAIN_NAME}-'
146+
@cat .cntb/instances.json | jq -r '.[] | .displayName' | grep '^${DOMAIN_NAME}_'
147+
148+
.PHONY: get-join-command
149+
get-join-command: ## Get the kubeadm join command for adding new nodes
150+
get-join-command: generate
151+
@ssh $(SSH_OPTS) root@$(CONTROL_PLANE_ENDPOINT) "kubeadm token create --print-join-command"
152+
153+
.PHONY: install-metallb
154+
install-metallb: ## Install MetalLB for load balancing
155+
install-metallb: generate
156+
@echo "Installing MetalLB..."
157+
@ssh $(SSH_OPTS) root@$(CONTROL_PLANE_ENDPOINT) "kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f https://raw.githubusercontent.com/metallb/metallb/v0.13.7/config/manifests/metallb-native.yaml"
158+
@echo "Waiting for MetalLB to be ready..."
159+
@ssh $(SSH_OPTS) root@$(CONTROL_PLANE_ENDPOINT) "kubectl --kubeconfig=/etc/kubernetes/admin.conf wait --namespace metallb-system --for=condition=ready pod --selector=app=metallb --timeout=90s"
160+
@echo "MetalLB installed successfully!"
161+
162+
.PHONY: install-kube-vip
163+
install-kube-vip: ## Install Kube-VIP for control plane high availability
164+
install-kube-vip: generate
165+
@echo "Installing Kube-VIP..."
166+
@ssh $(SSH_OPTS) root@$(CONTROL_PLANE_ENDPOINT) "kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f https://kube-vip.io/manifests/kube-vip-rbac.yaml"
167+
@echo "Kube-VIP installed successfully!"
60168

61169
.PHONY: test
62170
test: ## Test basic cluster functionalities

‎README.md‎

Lines changed: 55 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# High Availability but Lean Kubernetes Cluster
22

3-
A High Availability, frugal Kubernetes cluster using [Kubespray](https://github.com/kubernetes-sigs/kubespray) optimized for Contabo VPS. Deploy enterprise-grade Kubernetes at 1/10th the cost of managed solutions.
3+
A High Availability, frugal Kubernetes cluster using [kubeadm](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/) optimized for Contabo VPS. Deploy enterprise-grade Kubernetes at 1/10th the cost of managed solutions.
44

55
## Key Features
66

@@ -98,32 +98,70 @@ graph TD
9898
```bash
9999
git clone https://github.com/ctnr.io/ha-lean-cluster.git
100100
cd ha-lean-cluster
101-
make login
102-
make generate
103-
make apply
101+
deno run -A api/cli.ts createCluster --domainName=ctnr.io
104102
```
105103

104+
## API-Based Cluster Management
105+
106+
This project uses a tRPC-based API for managing your Kubernetes cluster. You can interact with the API using the provided CLI tool:
107+
108+
```bash
109+
# Create a new cluster based on available Contabo nodes
110+
deno run -A api/cli.ts createCluster --domainName=ctnr.io --k8sVersion=1.31.4 --cni=calico
111+
112+
# Add a control plane node to the cluster
113+
deno run -A api/cli.ts addControlPlane --domainName=ctnr.io --nodeName=ctnr.io_control-plane-1
114+
115+
# Add a worker node to the cluster
116+
deno run -A api/cli.ts addWorker --domainName=ctnr.io --nodeName=ctnr.io_worker-0
117+
118+
# Add an etcd node to the cluster (requires manual configuration)
119+
deno run -A api/cli.ts addEtcd --domainName=ctnr.io --nodeName=ctnr.io_etcd-0
120+
121+
# Remove a node from the cluster
122+
deno run -A api/cli.ts removeNode --domainName=ctnr.io --nodeName=ctnr.io_worker-0
123+
124+
# Upgrade the Kubernetes cluster
125+
deno run -A api/cli.ts upgradeCluster --domainName=ctnr.io --version=1.32.0 --packageRevision=00
126+
127+
# Reset the entire Kubernetes cluster
128+
deno run -A api/cli.ts resetCluster --domainName=ctnr.io --confirm=true
129+
130+
# List all nodes in the cluster
131+
deno run -A api/cli.ts listNodes --domainName=ctnr.io
132+
133+
# Get the kubeadm join command for adding new nodes
134+
deno run -A api/cli.ts getJoinCommand --domainName=ctnr.io
135+
```
136+
137+
### API Features
138+
139+
- **Automatic Node Provisioning**: When creating a cluster, the API checks for existing clusters with the same domain name and automatically claims available VPS instances.
140+
- **Type-Safe Operations**: All API operations are fully typed with input validation.
141+
- **Flexible Configuration**: Customize Kubernetes version, CNI, network CIDRs, and more.
142+
- **Error Handling**: Comprehensive error handling with detailed error messages.
143+
106144
## Hardware Requirements
107145

108146
- **Minimal**: 1 Contabo VPS S (4 vCPU, 8GB RAM) - €4.99/month
109147
- **Recommended**: 3 Contabo VPS S instances - €14.97/month
110148

111149
## Configuration & Node Naming
112150

113-
- **Core Files**: `inventory.ini.ts`, `kubeconfig.yml.ts`, `group_vars/k8s_cluster/k8s-cluster.yml.ts`, `group_vars/k8s_cluster/addons.yml.ts`
114-
- **Node Naming**: Each node name must start with the domain name prefix (`ctnr.io-`) followed by the role:
115-
- Control Plane: `<domain-name>-control-plane-X[-etcd][-worker]`
116-
- ETCD: `<domain-name>-etcd-X`
117-
- Worker: `<domain-name>-worker-X[-etcd]`
118-
- Examples: `<domain-name>-control-plane-0-etcd`, `<domain-name>-worker-0`, `<domain-name>-etcd-0`
151+
- **Core Files**: `kubeconfig.yml.ts`, `helpers.ts`
152+
- **Node Naming**: Each node name must start with the domain name prefix (`ctnr.io_`) followed by the role:
153+
- Control Plane: `<domain-name>_control-plane-X[_etcd][_worker]`
154+
- ETCD: `<domain-name>_etcd-X`
155+
- Worker: `<domain-name>_worker-X[_etcd]`
156+
- Examples: `<domain-name>_control-plane-0_etcd`, `<domain-name>_worker-0`, `<domain-name>_etcd-0`
119157

120158
### ⚠️ Important Warning
121159

122-
**DO NOT change the first control plane node (<domain-name>-control-plane-0) without understanding the implications!**
160+
**DO NOT change the first control plane node (<domain-name>_control-plane-0) without understanding the implications!**
123161

124162
The first control plane node is critical for cluster stability. Modifying or removing it incorrectly can cause the entire cluster to fail. If you need to replace the first control plane node, follow these steps:
125163

126-
1. Rename your current nodes so that `<domain-name>-control-plane-1` becomes `<domain-name>-control-plane-0` and vice versa
164+
1. Rename your current nodes so that `<domain-name>_control-plane-1` becomes `<domain-name>_control-plane-0` and vice versa
127165
2. Apply the configuration
128166
3. Only then remove the original control plane node
129167

@@ -132,11 +170,15 @@ The first control plane node is critical for cluster stability. Modifying or rem
132170
The project includes automated tests to verify that key components are working correctly:
133171

134172
- **Ingress Testing**: Tests that the Nginx ingress controller is properly configured and can route traffic to services
135-
- Run with: `make test-ingress` or `deno test -A tests/ingress_test.ts`
173+
- Run with: `deno test -A tests/ingress_test.ts`
136174
- The test creates a test namespace, deployment, service, and ingress with a custom domain name
137175
- It verifies connectivity using host-based routing with the domain name from your configuration
138176
- Test fixtures are located in `tests/fixtures/` directory
139177

178+
- **ETCD Testing**: Tests that the etcd cluster is healthy and functioning correctly
179+
- Run with: `deno test -A tests/etcd_test.ts`
180+
- Verifies etcd cluster health, member list, alarms, and response time
181+
140182
## Next Steps & Roadmap
141183

142184
- **Next**: Implement volume provisioning, configure auto-scaling, deploy workloads, add monitoring

‎api/caller.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
import { appRouter } from "./router/index.ts";
2+
import { trpc } from "./trpc.ts";
3+
4+
// Create a tRPC caller for direct procedure calls
5+
export const caller = trpc.createCallerFactory(appRouter)({});

‎api/cli.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
import { trpc } from "./trpc.ts";
2+
import { appRouter } from "./router/index.ts";
3+
import { createCli } from "trpc-cli";
4+
5+
export const cli = createCli({
6+
router: appRouter,
7+
createCallerFactory: trpc.createCallerFactory,
8+
});
9+
10+
cli.run();

0 commit comments

Comments
 (0)