Skip to content

Commit 9accc8b

Browse files
shaun-nxshaun-nx
committed
Add nfr-ipv6-only workflow to test with sample deployment manifests
1 parent 4d667af commit 9accc8b

File tree

3 files changed

+293
-10
lines changed

3 files changed

+293
-10
lines changed

‎.github/workflows/nfr-ipv6-only.yml‎

Lines changed: 292 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,292 @@
1+
name: IPv6-Only Testing
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
version:
7+
description: Version of NGF under test
8+
required: true
9+
default: edge
10+
image_tag:
11+
description: Tag of the NGF and NGINX Docker images
12+
required: true
13+
default: edge
14+
type:
15+
description: Type of NGINX image to test
16+
required: true
17+
default: both
18+
type: choice
19+
options: [oss, plus, both]
20+
schedule:
21+
- cron: "0 16 1,15 * *" # Run on the 1st and 15th of every month at 16:00 UTC
22+
defaults:
23+
run:
24+
shell: bash
25+
26+
env:
27+
PLUS_USAGE_ENDPOINT: ${{ secrets.JWT_PLUS_REPORTING_ENDPOINT }}
28+
29+
permissions:
30+
contents: read
31+
32+
jobs:
33+
ipv6-only-tests:
34+
name: Run IPv6-Only Tests
35+
runs-on: ubuntu-24.04
36+
if: ${{ !github.event.pull_request.head.repo.fork || inputs.image != 'plus' }}
37+
env:
38+
DOCKER_BUILD_SUMMARY: false
39+
steps:
40+
- name: Checkout Repository
41+
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
42+
43+
### Authenticate to GCP and set up gcloud, kubectl, and Docker
44+
- name: Authenticate to Google Cloud
45+
id: auth
46+
uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0
47+
with:
48+
token_format: access_token
49+
workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY }}
50+
service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
51+
52+
- name: Login to GAR
53+
uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0
54+
with:
55+
registry: us-docker.pkg.dev
56+
username: oauth2accesstoken
57+
password: ${{ steps.auth.outputs.access_token }}
58+
59+
- name: Set up Cloud SDK
60+
uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3.0.1
61+
with:
62+
project_id: ${{ secrets.GCP_PROJECT_ID }}
63+
install_components: kubectl
64+
###
65+
66+
- name: Setup dotenv file
67+
working-directory: ./tests/scripts
68+
run: |
69+
echo "RESOURCE_NAME=nfr-tests-${{ github.run_id }}-${{ matrix.type }}" >> vars.env
70+
echo "TAG=${{ needs.vars.outputs.image_tag }}" >> vars.env
71+
echo "PREFIX=ghcr.io/nginx/nginx-gateway-fabric" >> vars.env
72+
echo "NGINX_PREFIX=ghcr.io/nginx/nginx-gateway-fabric/nginx" >> vars.env
73+
echo "NGINX_PLUS_PREFIX=us-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/nginx-gateway-fabric/nginx-plus" >> vars.env
74+
echo "GKE_CLUSTER_NAME=nfr-tests-${{ github.run_id }}-${{ matrix.type }}" >> vars.env
75+
echo "GKE_CLUSTER_ZONE=us-west1-b" >> vars.env
76+
echo "GKE_CLUSTER_REGION=us-west1" >> vars.env
77+
echo "GKE_PROJECT=${{ secrets.GCP_PROJECT_ID }}" >> vars.env
78+
echo "GKE_SVC_ACCOUNT=${{ secrets.GCP_SERVICE_ACCOUNT }}" >> vars.env
79+
echo "GKE_NODES_SERVICE_ACCOUNT=${{ secrets.GKE_NODES_SERVICE_ACCOUNT }}" >> vars.env
80+
echo "NETWORK_TAGS=nfr-tests-${{ github.run_id }}-${{ matrix.type }}" >> vars.env
81+
echo "NGF_BRANCH=${{ github.ref_name }}" >> vars.env
82+
echo "SOURCE_IP_RANGE=$(curl -sS -4 icanhazip.com)/32" >> vars.env
83+
echo "ADD_VM_IP_AUTH_NETWORKS=true" >> vars.env
84+
echo "PLUS_ENABLED=${{ matrix.type == 'plus' }}" >> vars.env
85+
echo "GINKGO_LABEL=" >> vars.env
86+
echo "NGF_VERSION=${{ needs.vars.outputs.version }}" >> vars.env
87+
echo "GKE_NUM_NODES=1" >> vars.env
88+
echo "GKE_MACHINE_TYPE=n2d-standard-16" >> vars.env
89+
echo "IPV6_ENABLED=true" >> vars.env
90+
echo "PLUS_USAGE_ENDPOINT=${{ secrets.JWT_PLUS_REPORTING_ENDPOINT }}" >> vars.env
91+
- name: Setup license file for plus
92+
if: matrix.type == 'plus'
93+
env:
94+
PLUS_LICENSE: ${{ secrets.JWT_PLUS_REPORTING }}
95+
run: echo "${PLUS_LICENSE}" > license.jwt
96+
97+
- name: Create GKE cluster
98+
working-directory: ./tests
99+
run: make create-gke-cluster CI=true
100+
101+
- name: Create and setup VM
102+
working-directory: ./tests
103+
run: make create-and-setup-vm
104+
105+
- name: Create and setup Router
106+
working-directory: ./tests
107+
run: make create-gke-router || true
108+
109+
- name: Setup Golang Environment
110+
uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
111+
with:
112+
go-version: stable
113+
114+
- name: Set GOPATH
115+
run: echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
116+
117+
- name: Docker Buildx
118+
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
119+
120+
- name: NGF Docker meta
121+
id: ngf-meta
122+
uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0
123+
with:
124+
images: |
125+
name=ghcr.io/nginx/nginx-gateway-fabric
126+
tags: |
127+
type=semver,pattern={{version}}
128+
type=schedule
129+
type=edge
130+
type=ref,event=pr
131+
type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }}
132+
133+
- name: NGINX Docker meta
134+
id: nginx-meta
135+
uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0
136+
with:
137+
images: |
138+
name=ghcr.io/nginx/nginx-gateway-fabric/${{ inputs.image == 'plus' && 'nginx-plus' || inputs.image }}
139+
tags: |
140+
type=semver,pattern={{version}}
141+
type=edge
142+
type=schedule
143+
type=ref,event=pr
144+
type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }}
145+
146+
# - name: Build binary
147+
# uses: goreleaser/goreleaser-action@e435ccd777264be153ace6237001ef4d979d3a7a # v6.4.0
148+
# with:
149+
# version: v2.11.2 # renovate: datasource=github-tags depName=goreleaser/goreleaser
150+
# args: build --single-target --snapshot --clean
151+
# env:
152+
# TELEMETRY_ENDPOINT: otel-collector-opentelemetry-collector.collector.svc.cluster.local:4317
153+
# TELEMETRY_ENDPOINT_INSECURE: "true"
154+
155+
# - name: Build NGF Docker Image
156+
# uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
157+
# with:
158+
# file: build/Dockerfile
159+
# tags: ${{ steps.ngf-meta.outputs.tags }}
160+
# context: "."
161+
# load: true
162+
# cache-from: type=gha,scope=ngf-ipv6
163+
# pull: true
164+
# target: goreleaser
165+
166+
# - name: Build NGINX Docker Image
167+
# uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
168+
# with:
169+
# file: build/Dockerfile${{ inputs.image == 'nginx' && '.nginx' || '' }}${{ inputs.image == 'plus' && '.nginxplus' || ''}}
170+
# tags: ${{ steps.nginx-meta.outputs.tags }}
171+
# context: "."
172+
# load: true
173+
# cache-from: type=gha,scope=${{ inputs.image }}-ipv6
174+
# pull: true
175+
# build-args: |
176+
# NJS_DIR=internal/controller/nginx/modules/src
177+
# NGINX_CONF_DIR=internal/controller/nginx/conf
178+
# BUILD_AGENT=gha
179+
180+
# - name: Deploy IPv6-Only Kubernetes
181+
# id: k8s
182+
# run: |
183+
# # Enable IPv6 and container network options
184+
# sudo sysctl -w net.ipv6.conf.all.disable_ipv6=0
185+
# sudo sysctl -w net.ipv6.conf.all.forwarding=1
186+
187+
# # Create IPv6-only kind cluster
188+
# kind create cluster \
189+
# --name ${{ github.run_id }}-ipv6 \
190+
# --image=kindest/node:${{ inputs.k8s-version }} \
191+
# --config=config/cluster/kind-cluster-ipv6-only.yaml
192+
193+
# # Load images into the cluster
194+
# kind load docker-image ${{ join(fromJSON(steps.ngf-meta.outputs.json).tags, ' ') }} ${{ join(fromJSON(steps.nginx-meta.outputs.json).tags, ' ') }} --name ${{ github.run_id }}-ipv6
195+
196+
- name: Install NGF with IPv6 Configuration
197+
run: |
198+
ngf_prefix=ghcr.io/nginx/nginx-gateway-fabric
199+
ngf_tag=${{ steps.ngf-meta.outputs.version }}
200+
201+
# Install with IPv6-specific configuration
202+
HELM_PARAMETERS="--set nginx.config.ipFamily=ipv6 --set nginx.service.type=ClusterIP" \
203+
make helm-install-local${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag}
204+
working-directory: ./tests
205+
206+
- name: Deploy Test Applications
207+
run: |
208+
kubectl apply -f tests/manifests/ipv6-test-app.yaml
209+
210+
- name: Wait for NGF and Applications to be Ready
211+
run: |
212+
echo "Waiting for NGF to be ready..."
213+
kubectl wait --for=condition=available --timeout=300s deployment/nginx-gateway -n nginx-gateway
214+
215+
echo "Waiting for test applications to be ready..."
216+
kubectl wait --for=condition=available --timeout=300s deployment/test-app-ipv6
217+
218+
- name: Deploy IPv6 Test Client
219+
run: |
220+
kubectl apply -f tests/manifests/test-client-ipv6.yaml
221+
kubectl wait --for=condition=ready --timeout=300s pod/ipv6-test-client
222+
223+
- name: Get NGF IPv6 Address
224+
id: ngf-address
225+
run: |
226+
# Get the NGF service IPv6 address
227+
NGF_IPV6=$(kubectl get service nginx-gateway -n nginx-gateway -o jsonpath='{.spec.clusterIP}')
228+
echo "NGF IPv6 Address: $NGF_IPV6"
229+
echo "ngf_ipv6=$NGF_IPV6" >> $GITHUB_OUTPUT
230+
231+
- name: Run IPv6 Connectivity Tests
232+
run: |
233+
echo "=== Running IPv6-Only Tests ==="
234+
235+
# Test 1: Basic connectivity test using test client pod
236+
echo "Test 1: Basic IPv6 connectivity"
237+
kubectl exec ipv6-test-client -- curl --version
238+
kubectl exec ipv6-test-client -- nslookup nginx-gateway.nginx-gateway.svc.cluster.local
239+
240+
# Test 2: Test NGF service directly via IPv6
241+
echo "Test 2: NGF Service IPv6 connectivity"
242+
kubectl exec ipv6-test-client -- curl -6 --connect-timeout 30 --max-time 60 -v \
243+
-H "Host: ipv6-test.example.com" \
244+
"http://[${{ steps.ngf-address.outputs.ngf_ipv6 }}]:80/" || echo "Direct NGF test failed"
245+
246+
# Test 3: Test via service DNS
247+
echo "Test 3: Service DNS IPv6 connectivity"
248+
kubectl exec ipv6-test-client -- curl -6 --connect-timeout 30 --max-time 60 -v \
249+
-H "Host: ipv6-test.example.com" \
250+
"http://nginx-gateway.nginx-gateway.svc.cluster.local:80/" || echo "Service DNS test failed"
251+
252+
- name: Validate IPv6-Only Configuration
253+
run: |
254+
echo "=== Validating IPv6-Only Configuration ==="
255+
256+
# Check NGF configuration
257+
echo "NGF Pod IPv6 addresses:"
258+
kubectl get pods -n nginx-gateway -o wide
259+
260+
echo "NGF Service configuration:"
261+
kubectl get service nginx-gateway -n nginx-gateway -o yaml
262+
263+
echo "Gateway and HTTPRoute status:"
264+
kubectl get gateway,httproute -A -o wide
265+
266+
echo "Test application service configuration:"
267+
kubectl get service test-app-ipv6-service -o yaml
268+
269+
- name: Collect Logs
270+
if: always()
271+
run: |
272+
echo "=== Collecting logs for debugging ==="
273+
echo "NGF Controller logs:"
274+
kubectl logs -n nginx-gateway deployment/nginx-gateway -c nginx-gateway-controller --tail=100 || true
275+
276+
echo "NGINX logs:"
277+
kubectl logs -n nginx-gateway deployment/nginx-gateway -c nginx --tail=100 || true
278+
279+
echo "Test client logs:"
280+
kubectl logs ipv6-test-client --tail=100 || true
281+
282+
echo "Cluster events:"
283+
kubectl get events --sort-by='.lastTimestamp' --all-namespaces --tail=50 || true
284+
285+
- name: Cleanup
286+
working-directory: ./tests
287+
if: always()
288+
run: |
289+
bash scripts/cleanup-vm.sh true
290+
bash scripts/cleanup-router.sh true
291+
make delete-gke-cluster
292+
rm -rf scripts/vars.env

‎.github/workflows/nfr.yml‎

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -145,15 +145,6 @@ jobs:
145145
PLUS_LICENSE: ${{ secrets.JWT_PLUS_REPORTING }}
146146
run: echo "${PLUS_LICENSE}" > license.jwt
147147

148-
# - name: TMP Cleanup conflicting resources
149-
# working-directory: ./tests
150-
# run: |
151-
# echo "tmp delete conflicting subnet"
152-
# gcloud compute forwarding-rules delete gk3-nfr-tests-17637219369-oss-b6387c3a-d2289ac6-pe --region=us-west1 --project=${{ secrets.GCP_PROJECT_ID }} --quiet || true
153-
# gcloud compute addresses delete gk3-nfr-tests-17637219369-oss-b6387c3a-d2289ac6-pe --region=us-west1 --project=${{ secrets.GCP_PROJECT_ID }} --quiet || true
154-
# gcloud compute networks subnets delete gke-nfr-tests-17637219369-oss-acf63488-pe-subnet --region=us-west1 --quiet || true
155-
# rm -rf scripts/vars.env
156-
157148
- name: Create GKE cluster
158149
working-directory: ./tests
159150
run: make create-gke-cluster CI=true

‎tests/scripts/create-and-setup-gcp-vm.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ if [ "${IPV6_ENABLED}" = "true" ]; then
2626
# --range=10.120.0.0/14
2727

2828
NETWORK="us-ipv4-ipv6"
29-
# NETWORK_TIER="network-tier=PREMIUM"
29+
# NETWORK_TIER="ipv6-network-tier=PREMIUM" # This will work only if STACK_TYPE is IPV6_ONLY
3030
STACK_TYPE="IPV4_IPV6"
3131
fi
3232

0 commit comments

Comments
 (0)