Skip to content

Commit 05c3cc4

Browse files
committed
feat: gateway api support
1 parent 581cedd commit 05c3cc4

File tree

42 files changed

+1533
-137
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+1533
-137
lines changed

.github/actions/setup-ubuntu/action.yaml

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,10 @@ runs:
1515
- name: Install dependencies
1616
# Shell must explicitly specify the shell for each step. https://github.com/orgs/community/discussions/18597
1717
shell: bash
18-
run: sudo apt install -y pkg-config make direnv unzip lz4 wget curl npm jq pv coreutils libudev-dev
18+
run: |
19+
sudo apt-get update
20+
sudo apt-get install -y \
21+
pkg-config make direnv unzip lz4 wget curl npm jq pv coreutils libudev-dev
1922
- name: Setup npm
2023
uses: actions/setup-node@v4
2124
with:

.github/workflows/integration-tests.yaml

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,10 @@ jobs:
3535
- name: Install dependencies
3636
# Shell must explicitly specify the shell for each step. https://github.com/orgs/community/discussions/18597
3737
shell: bash
38-
run: sudo apt install -y pkg-config make direnv unzip lz4 wget curl jq pv coreutils libudev-dev
38+
run: |
39+
sudo apt-get update
40+
sudo apt-get install -y \
41+
pkg-config make direnv unzip lz4 wget curl jq pv coreutils libudev-dev
3942
- name: Detect required Go version
4043
working-directory: "${{ env.GOPATH }}/src/github.com/akash-network/provider"
4144
run: |

_docs/kustomize/akash-operator-hostname/cluster-roles.yaml

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,19 @@ rules:
2121
- delete
2222
- deletecollection
2323
- watch
24+
- apiGroups:
25+
- gateway.networking.k8s.io
26+
resources:
27+
- httproutes
28+
- gateways
29+
verbs:
30+
- get
31+
- list
32+
- create
33+
- update
34+
- delete
35+
- deletecollection
36+
- watch
2437
- apiGroups:
2538
- ""
2639
resources:
Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
---
2+
apiVersion: v1
3+
kind: ConfigMap
4+
metadata:
5+
name: operator-hostname
6+
data:
7+
k8s-manifest-ns: lease
8+
prune-interval: 600s
9+
ignore-list-entry-limit: "131072"
10+
web-refresh-interval: 5s
11+
retry-delay: 3s
12+
ignore-list-age-limit: 2613600s
13+
event-failure-limit: "3"
14+
ingress-mode: ingress
15+
gateway-name: akash-gateway
16+
gateway-namespace: akash-gateway
17+
18+

_docs/kustomize/akash-operator-hostname/deployment.yaml

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,21 @@ spec:
7070
configMapKeyRef:
7171
name: operator-hostname
7272
key: event-failure-limit
73+
- name: AP_INGRESS_MODE
74+
valueFrom:
75+
configMapKeyRef:
76+
name: operator-hostname
77+
key: ingress-mode
78+
- name: AP_GATEWAY_NAME
79+
valueFrom:
80+
configMapKeyRef:
81+
name: operator-hostname
82+
key: gateway-name
83+
- name: AP_GATEWAY_NAMESPACE
84+
valueFrom:
85+
configMapKeyRef:
86+
name: operator-hostname
87+
key: gateway-namespace
7388
- name: AP_POD_NAME
7489
valueFrom:
7590
fieldRef:

_docs/kustomize/akash-operator-hostname/kustomization.yaml

Lines changed: 1 addition & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3,19 +3,10 @@ apiVersion: kustomize.config.k8s.io/v1beta1
33
kind: Kustomization
44
namespace: akash-services
55
resources:
6+
- configmap.yaml
67
- deployment.yaml
78
- service.yaml
89
- ingress.yaml
910
- role-bindings.yaml
1011
- service-accounts.yaml
1112
- cluster-roles.yaml
12-
configMapGenerator:
13-
- name: operator-hostname
14-
literals:
15-
- k8s-manifest-ns=lease
16-
- prune-interval=600s
17-
- ignore-list-entry-limit=131072
18-
- web-refresh-interval=5s
19-
- retry-delay=3s
20-
- ignore-list-age-limit=2613600s
21-
- event-failure-limit=3

_run/kube/Makefile

Lines changed: 79 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,5 +46,83 @@ clean-kube:
4646
.PHONY: kube-deployments-rollout
4747
kube-deployments-rollout: #$(patsubst %, kube-deployment-rollout-%,$(KUSTOMIZE_INSTALLS))
4848

49+
.PHONY: kube-setup-hostname-operator-ingress
50+
kube-setup-hostname-operator-ingress:
51+
@echo "Configuring hostname operator for NGINX Ingress mode..."
52+
kubectl wait --for=condition=available deployment/operator-hostname -n akash-services --timeout=90s
53+
kubectl patch configmap operator-hostname -n akash-services --type=strategic --patch-file hostname-operator-ingress-patch.yaml
54+
kubectl rollout restart deployment/operator-hostname -n akash-services
55+
kubectl rollout status deployment/operator-hostname -n akash-services --timeout=60s
56+
4957
.PHONY: kube-setup-kube
50-
kube-setup-kube:
58+
kube-setup-kube: kube-setup-hostname-operator-ingress
59+
60+
.INTERMEDIATE: kube-cluster-create-kind-gateway
61+
kube-cluster-create-kind-gateway: $(KIND)
62+
$(KIND) create cluster --config kind-config-gateway.yaml --name "$(KIND_NAME)" --image "$(KIND_IMG)"
63+
64+
KUBE_CREATE_GATEWAY := $(AP_RUN_DIR)/.kube-create-gateway
65+
66+
$(KUBE_CREATE_GATEWAY): $(AP_RUN_DIR) kube-cluster-create-kind-gateway
67+
$(SETUP_KUBE) --crd=$(CRD_FILE) kind init "$(KUBE_SSH_NODES)"
68+
touch $@
69+
70+
.PHONY: kube-setup-ingress-gateway
71+
kube-setup-ingress-gateway:
72+
kubectl kustomize "https://github.com/nginx/nginx-gateway-fabric/config/crd/gateway-api/standard?ref=v2.3.0" | kubectl apply -f -
73+
helm install ngf oci://ghcr.io/nginx/charts/nginx-gateway-fabric \
74+
--create-namespace \
75+
-n nginx-gateway \
76+
--set nginx.service.type=NodePort \
77+
--set-json 'nginx.service.nodePorts=[{"port":31437,"listenerPort":80},{"port":31438,"listenerPort":443}]'
78+
kubectl apply -f "$(METALLB_CONFIG_PATH)"
79+
kubectl apply -f "$(METALLB_IP_CONFIG_PATH)"
80+
kubectl apply -f "$(METALLB_SERVICE_PATH)"
81+
kubectl apply -f gateway-resources.yaml
82+
@echo "Waiting for Gateway to be ready..."
83+
@kubectl wait --for=condition=Programmed gateway/akash-gateway -n akash-gateway --timeout=120s || true
84+
85+
.PHONY: kube-setup-hostname-operator-gateway
86+
kube-setup-hostname-operator-gateway:
87+
@echo "Configuring hostname operator for Gateway API mode..."
88+
kubectl wait --for=condition=available deployment/operator-hostname -n akash-services --timeout=90s
89+
kubectl patch configmap operator-hostname -n akash-services --type=strategic --patch-file hostname-operator-gateway-patch.yaml
90+
kubectl rollout restart deployment/operator-hostname -n akash-services
91+
kubectl rollout status deployment/operator-hostname -n akash-services --timeout=60s
92+
93+
.PHONY: kube-cluster-setup-gateway
94+
kube-cluster-setup-gateway: init \
95+
kube-prepare-images \
96+
$(KUBE_CREATE_GATEWAY) \
97+
kube-cluster-check-info \
98+
kube-setup-ingress-gateway \
99+
kube-upload-images \
100+
kustomize-init \
101+
kustomize-deploy-services \
102+
kube-deployments-rollout \
103+
kube-install-helm-charts \
104+
kube-setup-$(AP_RUN_NAME) \
105+
kube-setup-hostname-operator-gateway
106+
107+
.PHONY: provider-run-gateway
108+
provider-run-gateway:
109+
$(PROVIDER_SERVICES) run \
110+
--from "$(PROVIDER_KEY_NAME)" \
111+
--cluster-k8s \
112+
--gateway-listen-address "$(GATEWAY_HOST)" \
113+
--deployment-ingress-static-hosts true \
114+
--deployment-ingress-domain "$(GATEWAY_HOSTNAME)" \
115+
--cluster-node-port-quantity 100 \
116+
--cluster-public-hostname "$(GATEWAY_HOSTNAME)" \
117+
--bid-price-strategy "randomRange" \
118+
--deployment-runtime-class "none" \
119+
--ip-operator=true \
120+
--ingress-mode=gateway-api \
121+
--gateway-name=akash-gateway \
122+
--gateway-namespace=akash-gateway
123+
124+
# Override the standard kube-cluster-delete-kind to handle both marker files
125+
.PHONY: kube-cluster-delete
126+
kube-cluster-delete: $(KIND)
127+
$(KIND) delete cluster --name "$(KIND_NAME)"
128+
rm -rf $(KUBE_CREATE) $(KUBE_CREATE_GATEWAY)

_run/kube/README.md

Lines changed: 80 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,10 +53,12 @@ If at any time you'd like to start over with a fresh chain, simply run:
5353

5454
__t1 run__
5555
```sh
56-
make clean kind-cluster-clean
56+
make clean kube-cluster-delete
5757
make init
5858
```
5959

60+
Note: The same cleanup command works for both NGINX Ingress and Gateway API modes.
61+
6062
## Initialize
6163

6264
Start and initialize kind.
@@ -73,9 +75,37 @@ there may be a problem.
7375

7476
| Option | __t1 Step: 1__ | Explanation |
7577
|---------------------------------------------------|------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------|
76-
| Map random local port to port 80 of your workload | `make kind-cluster-setup` | This is less error-prone, but makes it difficult to access your app through the browser. |
78+
| Map random local port to port 80 of your workload | `make kind-cluster-setup` | This is less error-prone, but makes it difficult to access your app through the browser. Configures hostname operator for NGINX Ingress mode. |
7779
| Map localhost port 80 to workload | `KIND_CONFIG=kind-config-80.yaml make kind-cluster-create` | If anything else is listening on port 80 (any other web server), this method will fail. If it does succeed, you will be able to browse your app from the browser. |
7880

81+
### Gateway API Setup (Optional)
82+
83+
As an alternative to NGINX Ingress, you can use Kubernetes Gateway API. NGINX Ingress will be EOL by March 2026, so Gateway API is the recommended approach for new deployments.
84+
85+
| Option | __t1 Step: 1 (Gateway API)__ | Explanation |
86+
|------------------------------------------|----------------------------------|--------------------------------------------------------------------------------------------------|
87+
| Gateway API with NGINX Gateway Fabric | `make kube-cluster-setup-gateway`| Creates kind cluster with Gateway API support, installs NGINX Gateway Fabric, and sets up all resources. Configures hostname operator for Gateway API mode. Maps ports 8080 (HTTP) and 8443 (HTTPS). |
88+
89+
This comprehensive setup target:
90+
- Creates a kind cluster with port mappings for HTTP (8080) and HTTPS (8443)
91+
- Installs Gateway API CRDs (v1)
92+
- Installs NGINX Gateway Fabric controller
93+
- Configures NodePort service for ports 31437 (HTTP) and 31438 (HTTPS)
94+
- Creates the Gateway resource (`akash-gateway`)
95+
- Configures the hostname operator for Gateway API mode
96+
- Sets up all necessary Akash operators and services
97+
98+
You can verify the Gateway is ready:
99+
```sh
100+
kubectl get gateway akash-gateway -n akash-gateway
101+
```
102+
103+
Expected output:
104+
```
105+
NAME CLASS CLUSTER-IP ADDRESS PROGRAMMED AGE
106+
akash-gateway nginx 10.96.x.x 10.96.x.x True 1m
107+
```
108+
79109
## Build Akash binaries and initialize network
80110

81111
Initialize keys and accounts:
@@ -131,17 +161,35 @@ make query-provider
131161
To run Provider as a simple binary connecting to the cluster, in a third terminal, run the command:
132162

133163
### __t3 Step: 5__
164+
165+
**With NGINX Ingress (default):**
134166
```sh
135167
make provider-run
136168
```
137169

170+
**With Gateway API:**
171+
```sh
172+
make provider-run-gateway
173+
```
174+
175+
The Gateway API mode uses the following flags:
176+
- `--ingress-mode=gateway-api` - Enable Gateway API instead of NGINX Ingress
177+
- `--gateway-name=akash-gateway` - Name of the Gateway resource to use
178+
- `--gateway-namespace=akash-gateway` - Namespace where the Gateway exists
179+
138180
Query the provider service gateway for its status:
139181

140182
__t1 status__
141183
```sh
142184
make provider-status
143185
```
144186

187+
When using Gateway API mode, deployments will create `HTTPRoute` resources instead of `Ingress` resources. You can verify this with:
188+
189+
```sh
190+
kubectl get httproutes -A
191+
```
192+
145193
## Create a deployment
146194

147195
Create a deployment from the `main` account with:
@@ -250,6 +298,36 @@ Between the first and second step, the prior deployment's containers will contin
250298

251299
Akash Groups are translated into Kubernetes Deployments, this means that only a few fields from the Akash SDL are mutable. For example `image`, `command`, `args`, `env` and exposed ports can be modified, but compute resources and placement criteria cannot.
252300

301+
## Gateway API vs NGINX Ingress
302+
303+
### Comparison
304+
305+
| Feature | NGINX Ingress (default) | Gateway API |
306+
|---------|------------------------|-------------|
307+
| Resource Type | `Ingress` | `HTTPRoute` |
308+
| Class/Reference | `IngressClass: akash-ingress-class` | `Gateway: akash-gateway` |
309+
| Annotations | `nginx.ingress.kubernetes.io/*` | `nginx.org/*` |
310+
| EOL Status | March 2026 | Current standard |
311+
312+
### HTTP Options Translation
313+
314+
The provider automatically translates HTTP options from the SDL to the appropriate annotations:
315+
316+
| SDL Option | NGINX Ingress Annotation | Gateway API Annotation |
317+
|------------|-------------------------|------------------------|
318+
| Read Timeout | `nginx.ingress.kubernetes.io/proxy-read-timeout` | `nginx.org/proxy-read-timeout` |
319+
| Send Timeout | `nginx.ingress.kubernetes.io/proxy-send-timeout` | `nginx.org/proxy-send-timeout` |
320+
| Max Body Size | `nginx.ingress.kubernetes.io/proxy-body-size` | `nginx.org/client-max-body-size` |
321+
| Next Upstream Tries | `nginx.ingress.kubernetes.io/proxy-next-upstream-tries` | `nginx.org/proxy-next-upstream-tries` |
322+
323+
### Cleanup
324+
325+
To remove the cluster (works for both NGINX Ingress and Gateway API):
326+
327+
```sh
328+
make kube-cluster-delete
329+
```
330+
253331
## Terminate lease
254332

255333
There are a number of ways that a lease can be terminated.

_run/kube/gateway-resources.yaml

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
---
2+
apiVersion: v1
3+
kind: Namespace
4+
metadata:
5+
name: akash-gateway
6+
---
7+
apiVersion: gateway.networking.k8s.io/v1
8+
kind: Gateway
9+
metadata:
10+
name: akash-gateway
11+
namespace: akash-gateway
12+
spec:
13+
gatewayClassName: nginx
14+
listeners:
15+
- name: http
16+
port: 80
17+
protocol: HTTP
18+
allowedRoutes:
19+
namespaces:
20+
from: All
Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
---
2+
apiVersion: v1
3+
kind: ConfigMap
4+
metadata:
5+
name: operator-hostname
6+
data:
7+
ingress-mode: gateway-api
8+
gateway-name: akash-gateway
9+
gateway-namespace: akash-gateway
10+
11+

0 commit comments

Comments
 (0)