diff --git a/.github/workflows/ipv6-only.yml b/.github/workflows/ipv6-only.yml new file mode 100644 index 0000000000..c217a41242 --- /dev/null +++ b/.github/workflows/ipv6-only.yml @@ -0,0 +1,227 @@ +name: IPv6-Only Testing + +on: + workflow_call: + inputs: + image: + required: true + type: string + k8s-version: + required: true + type: string + +defaults: + run: + shell: bash + +env: + PLUS_USAGE_ENDPOINT: ${{ secrets.JWT_PLUS_REPORTING_ENDPOINT }} + +permissions: + contents: read + +jobs: + ipv6-only-tests: + name: Run IPv6-Only Tests + runs-on: ubuntu-24.04 + if: ${{ !github.event.pull_request.head.repo.fork || inputs.image != 'plus' }} + env: + DOCKER_BUILD_SUMMARY: false + steps: + - name: Checkout Repository + uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0 + with: + fetch-depth: 0 + + - name: Configure GOPROXY + id: goproxy + run: | + if [[ "${{ secrets.ARTIFACTORY_USER }}" == "" ]]; then + GOPROXY_VALUE="direct" + else + GOPROXY_VALUE="https://${{ secrets.ARTIFACTORY_USER }}:${{ secrets.ARTIFACTORY_TOKEN }}@${{ secrets.ARTIFACTORY_DEV_ENDPOINT }}" + fi + echo "GOPROXY=${GOPROXY_VALUE}" >> $GITHUB_ENV + + - name: Setup Golang Environment + uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0 + with: + go-version: stable + + - name: Set GOPATH + run: echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV + + - name: Docker Buildx + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + + - name: NGF Docker meta + id: ngf-meta + uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0 + with: + images: | + name=ghcr.io/nginx/nginx-gateway-fabric + tags: | + type=semver,pattern={{version}} + type=schedule + type=edge + type=ref,event=pr + type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} + + - name: NGINX Docker meta + id: nginx-meta + uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0 + with: + images: | + name=ghcr.io/nginx/nginx-gateway-fabric/${{ inputs.image == 'plus' && 'nginx-plus' || inputs.image }} + tags: | + type=semver,pattern={{version}} + type=edge + type=schedule + type=ref,event=pr + type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} + + - name: Build binary + uses: goreleaser/goreleaser-action@e435ccd777264be153ace6237001ef4d979d3a7a # v6.4.0 + with: + version: v2.12.0 # renovate: datasource=github-tags depName=goreleaser/goreleaser + args: + build --single-target --snapshot --clean --config=goreleaser.yml + env: + TELEMETRY_ENDPOINT: otel-collector-opentelemetry-collector.collector.svc.cluster.local:4317 + TELEMETRY_ENDPOINT_INSECURE: "true" + + - name: Build NGF Docker Image + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + file: build/Dockerfile + tags: ${{ steps.ngf-meta.outputs.tags }} + context: "." + load: true + cache-from: type=gha,scope=ngf + pull: true + target: goreleaser + + - name: Build NGINX Docker Image + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + file: build/Dockerfile${{ inputs.image == 'nginx' && '.nginx' || '' }}${{ inputs.image == 'plus' && '.nginxplus' || ''}} + tags: ${{ steps.nginx-meta.outputs.tags }} + context: "." + load: true + cache-from: type=gha,scope=${{ inputs.image }}-ipv6 + pull: true + build-args: | + NJS_DIR=internal/controller/nginx/modules/src + NGINX_CONF_DIR=internal/controller/nginx/conf + BUILD_AGENT=gha + + - name: Setup license file for plus + if: ${{ inputs.image == 'plus' }} + env: + PLUS_LICENSE: ${{ secrets.JWT_PLUS_REPORTING }} + run: echo "${PLUS_LICENSE}" > license.jwt + + - name: Deploy IPv6-Only Kubernetes + id: k8s + run: | + # Enable IPv6 and container network options + # sudo sysctl -w net.ipv6.conf.all.disable_ipv6=0 + # sudo sysctl -w net.ipv6.conf.all.forwarding=1 + + # Create IPv6-only kind cluster + kind create cluster \ + --name ${{ github.run_id }}-ipv6 \ + --image=kindest/node:${{ inputs.k8s-version }} \ + --config=config/cluster/kind-ipv6-only.yaml + + # Load images into the cluster + kind load docker-image ${{ join(fromJSON(steps.ngf-meta.outputs.json).tags, ' ') }} ${{ join(fromJSON(steps.nginx-meta.outputs.json).tags, ' ') }} --name ${{ github.run_id }}-ipv6 + + # Verify nodes are ipv6 only + kubectl get nodes -o wide + + - name: Install NGF with IPv6 Configuration + run: | + ngf_prefix=ghcr.io/nginx/nginx-gateway-fabric + ngf_tag=${{ steps.ngf-meta.outputs.version }} + # Install with IPv6-specific configuration + CLUSTER_NAME=${{ github.run_id }}-ipv6 \ + HELM_PARAMETERS="--set nginx.config.ipFamily=ipv6 --set nginx.service.type=ClusterIP" \ + make helm-install-local${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} + + working-directory: ./tests + + - name: Deploy Test Applications + run: | + kubectl apply -f tests/manifests/ipv6-test-app.yaml + + - name: Wait for NGF and Applications to be Ready + run: | + echo "Waiting for NGF to be ready..." + kubectl wait --for=condition=available --timeout=300s deployment/nginx-gateway -n nginx-gateway + echo "Waiting for test applications to be ready..." + kubectl wait --for=condition=available --timeout=300s deployment/test-app-ipv6 + + - name: Deploy IPv6 Test Client + run: | + kubectl apply -f tests/manifests/test-client-ipv6.yaml + kubectl wait --for=condition=ready --timeout=300s pod/ipv6-test-client + + - name: Get NGF IPv6 Address + id: ngf-address + run: | + # Get the NGF service IPv6 address + NGF_IPV6=$(kubectl get service nginx-gateway -n nginx-gateway -o jsonpath='{.spec.clusterIP}') + echo "NGF IPv6 Address: $NGF_IPV6" + echo "ngf_ipv6=$NGF_IPV6" >> $GITHUB_OUTPUT + + - name: Run IPv6 Connectivity Tests + run: | + echo "=== Running IPv6-Only Tests ===" + # Test 1: Basic connectivity test using test client pod + echo "Test 1: Basic IPv6 connectivity" + kubectl exec ipv6-test-client -- curl --version + kubectl exec ipv6-test-client -- nslookup nginx-gateway.nginx-gateway.svc.cluster.local + # Test 2: Test NGF service directly via IPv6 + echo "Test 2: NGF Service IPv6 connectivity" + kubectl exec ipv6-test-client -- curl -6 --connect-timeout 30 --max-time 60 -v \ + -H "Host: ipv6-test.example.com" \ + "http://[${{ steps.ngf-address.outputs.ngf_ipv6 }}]:80/" || echo "Direct NGF test failed" + # Test 3: Test via service DNS + echo "Test 3: Service DNS IPv6 connectivity" + kubectl exec ipv6-test-client -- curl -6 --connect-timeout 30 --max-time 60 -v \ + -H "Host: ipv6-test.example.com" \ + "http://nginx-gateway.nginx-gateway.svc.cluster.local:80/" || echo "Service DNS test failed" + + - name: Validate IPv6-Only Configuration + run: | + echo "=== Validating IPv6-Only Configuration ===" + # Check NGF configuration + echo "NGF Pod IPv6 addresses:" + kubectl get pods -n nginx-gateway -o wide + echo "NGF Service configuration:" + kubectl get service nginx-gateway -n nginx-gateway -o yaml + echo "Gateway and HTTPRoute status:" + kubectl get gateway,httproute -A -o wide + echo "Test application service configuration:" + kubectl get service test-app-ipv6-service -o yaml + + - name: Collect Logs + if: always() + run: | + echo "=== Collecting logs for debugging ===" + echo "NGF Controller logs:" + kubectl logs -n nginx-gateway deployment/nginx-gateway -c nginx-gateway-controller --tail=100 || true + echo "NGINX logs:" + kubectl logs -n nginx-gateway deployment/nginx-gateway -c nginx --tail=100 || true + echo "Test client logs:" + kubectl logs ipv6-test-client --tail=100 || true + echo "Cluster events:" + kubectl get events --sort-by='.lastTimestamp' --all-namespaces --tail=50 || true + + - name: Cleanup + if: always() + env: + RUN_ID: ${{ github.run_id }} + run: | + kind delete cluster --name "$RUN_ID-ipv6" || true diff --git a/.github/workflows/nfr-ipv6-only.yml b/.github/workflows/nfr-ipv6-only.yml new file mode 100644 index 0000000000..0f198a65f8 --- /dev/null +++ b/.github/workflows/nfr-ipv6-only.yml @@ -0,0 +1,319 @@ +name: IPv6-Only Testing + +on: + workflow_dispatch: + inputs: + version: + description: Version of NGF under test + required: true + default: edge + image_tag: + description: Tag of the NGF and NGINX Docker images + required: true + default: edge + type: + description: Type of NGINX image to test + required: true + default: both + type: choice + options: [oss, plus, both] + schedule: + - cron: "0 16 1,15 * *" # Run on the 1st and 15th of every month at 16:00 UTC +defaults: + run: + shell: bash + +env: + PLUS_USAGE_ENDPOINT: ${{ secrets.JWT_PLUS_REPORTING_ENDPOINT }} + +permissions: + contents: read + +jobs: + vars: + name: Set up vars + runs-on: ubuntu-24.04 + outputs: + version: ${{ github.event.inputs.version || 'edge' }} + image_tag: ${{ github.event.inputs.image_tag || 'edge' }} + types: ${{ steps.var.outputs.types }} + permissions: + contents: read + steps: + - name: Set vars + id: var + run: | + if ${{ github.event.inputs.type == 'both' || github.event_name == 'schedule' }}; then + echo 'types=["oss","plus"]' >> $GITHUB_OUTPUT + else + echo 'types=["${{ github.event.inputs.type }}"]' >> $GITHUB_OUTPUT + fi + + setup-and-run-tests: + name: Run IPv6-Only Tests + runs-on: ubuntu-24.04 + permissions: + contents: read + id-token: write # needed for authenticating to GCP + needs: vars + strategy: + fail-fast: false + matrix: + type: ${{ fromJson(needs.vars.outputs.types) }} + if: ${{ !github.event.pull_request.head.repo.fork || inputs.image != 'plus' }} + env: + DOCKER_BUILD_SUMMARY: false + steps: + - name: Checkout Repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + ### Authenticate to GCP and set up gcloud, kubectl, and Docker + - name: Authenticate to Google Cloud + id: auth + uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0 + with: + token_format: access_token + workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY }} + service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }} + + - name: Login to GAR + uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0 + with: + registry: us-docker.pkg.dev + username: oauth2accesstoken + password: ${{ steps.auth.outputs.access_token }} + + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3.0.1 + with: + project_id: ${{ secrets.GCP_PROJECT_ID }} + install_components: kubectl + ### + + - name: Setup dotenv file + working-directory: ./tests/scripts + run: | + echo "RESOURCE_NAME=nfr-tests-${{ github.run_id }}-${{ matrix.type }}" >> vars.env + echo "TAG=${{ needs.vars.outputs.image_tag }}" >> vars.env + echo "PREFIX=ghcr.io/nginx/nginx-gateway-fabric" >> vars.env + echo "NGINX_PREFIX=ghcr.io/nginx/nginx-gateway-fabric/nginx" >> vars.env + echo "NGINX_PLUS_PREFIX=us-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/nginx-gateway-fabric/nginx-plus" >> vars.env + echo "GKE_CLUSTER_NAME=nfr-tests-${{ github.run_id }}-${{ matrix.type }}" >> vars.env + echo "GKE_CLUSTER_ZONE=us-west1-b" >> vars.env + echo "GKE_CLUSTER_REGION=us-west1" >> vars.env + echo "GKE_PROJECT=${{ secrets.GCP_PROJECT_ID }}" >> vars.env + echo "GKE_SVC_ACCOUNT=${{ secrets.GCP_SERVICE_ACCOUNT }}" >> vars.env + echo "GKE_NODES_SERVICE_ACCOUNT=${{ secrets.GKE_NODES_SERVICE_ACCOUNT }}" >> vars.env + echo "NETWORK_TAGS=nfr-tests-${{ github.run_id }}-${{ matrix.type }}" >> vars.env + echo "NGF_BRANCH=${{ github.ref_name }}" >> vars.env + echo "SOURCE_IP_RANGE=$(curl -sS -4 icanhazip.com)/32" >> vars.env + echo "ADD_VM_IP_AUTH_NETWORKS=true" >> vars.env + echo "PLUS_ENABLED=${{ matrix.type == 'plus' }}" >> vars.env + echo "GINKGO_LABEL=" >> vars.env + echo "NGF_VERSION=${{ needs.vars.outputs.version }}" >> vars.env + echo "GKE_NUM_NODES=1" >> vars.env + echo "GKE_MACHINE_TYPE=n2d-standard-16" >> vars.env + echo "IPV6_ENABLED=true" >> vars.env + echo "PLUS_USAGE_ENDPOINT=${{ secrets.JWT_PLUS_REPORTING_ENDPOINT }}" >> vars.env + - name: Setup license file for plus + if: matrix.type == 'plus' + env: + PLUS_LICENSE: ${{ secrets.JWT_PLUS_REPORTING }} + run: echo "${PLUS_LICENSE}" > license.jwt + + - name: Create GKE cluster + working-directory: ./tests + run: make create-gke-cluster CI=true + + - name: Create and setup VM + working-directory: ./tests + run: make create-and-setup-vm + + - name: Create and setup Router + working-directory: ./tests + run: make create-gke-router || true + + - name: Setup Golang Environment + uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 + with: + go-version: stable + + - name: Set GOPATH + run: echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV + + - name: Docker Buildx + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + + - name: NGF Docker meta + id: ngf-meta + uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0 + with: + images: | + name=ghcr.io/nginx/nginx-gateway-fabric + tags: | + type=semver,pattern={{version}} + type=schedule + type=edge + type=ref,event=pr + type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} + + - name: NGINX Docker meta + id: nginx-meta + uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0 + with: + images: | + name=ghcr.io/nginx/nginx-gateway-fabric/${{ inputs.image_tag == 'plus' && 'nginx-plus' || inputs.image_tag§ }} + tags: | + type=semver,pattern={{version}} + type=edge + type=schedule + type=ref,event=pr + type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} + + # - name: Build binary + # uses: goreleaser/goreleaser-action@e435ccd777264be153ace6237001ef4d979d3a7a # v6.4.0 + # with: + # version: v2.11.2 # renovate: datasource=github-tags depName=goreleaser/goreleaser + # args: build --single-target --snapshot --clean + # env: + # TELEMETRY_ENDPOINT: otel-collector-opentelemetry-collector.collector.svc.cluster.local:4317 + # TELEMETRY_ENDPOINT_INSECURE: "true" + + # - name: Build NGF Docker Image + # uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + # with: + # file: build/Dockerfile + # tags: ${{ steps.ngf-meta.outputs.tags }} + # context: "." + # load: true + # cache-from: type=gha,scope=ngf-ipv6 + # pull: true + # target: goreleaser + + # - name: Build NGINX Docker Image + # uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + # with: + # file: build/Dockerfile${{ inputs.image == 'nginx' && '.nginx' || '' }}${{ inputs.image == 'plus' && '.nginxplus' || ''}} + # tags: ${{ steps.nginx-meta.outputs.tags }} + # context: "." + # load: true + # cache-from: type=gha,scope=${{ inputs.image }}-ipv6 + # pull: true + # build-args: | + # NJS_DIR=internal/controller/nginx/modules/src + # NGINX_CONF_DIR=internal/controller/nginx/conf + # BUILD_AGENT=gha + + # - name: Deploy IPv6-Only Kubernetes + # id: k8s + # run: | + # # Enable IPv6 and container network options + # sudo sysctl -w net.ipv6.conf.all.disable_ipv6=0 + # sudo sysctl -w net.ipv6.conf.all.forwarding=1 + + # # Create IPv6-only kind cluster + # kind create cluster \ + # --name ${{ github.run_id }}-ipv6 \ + # --image=kindest/node:${{ inputs.k8s-version }} \ + # --config=config/cluster/kind-cluster-ipv6-only.yaml + + # # Load images into the cluster + # kind load docker-image ${{ join(fromJSON(steps.ngf-meta.outputs.json).tags, ' ') }} ${{ join(fromJSON(steps.nginx-meta.outputs.json).tags, ' ') }} --name ${{ github.run_id }}-ipv6 + + - name: Install NGF with IPv6 Configuration + run: | + ngf_prefix=ghcr.io/nginx/nginx-gateway-fabric + ngf_tag=${{ steps.ngf-meta.outputs.version }} + + # Install with IPv6-specific configuration + # HELM_PARAMETERS="--set nginx.config.ipFamily=ipv6 --set nginx.service.type=ClusterIP" \ + # make helm-install-local${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} + working-directory: ./tests + + - name: Deploy Test Applications + run: | + kubectl apply -f tests/manifests/ipv6-test-app.yaml + + - name: Wait for NGF and Applications to be Ready + run: | + echo "Waiting for NGF to be ready..." + kubectl wait --for=condition=available --timeout=300s deployment/nginx-gateway -n nginx-gateway + + echo "Waiting for test applications to be ready..." + kubectl wait --for=condition=available --timeout=300s deployment/test-app-ipv6 + + - name: Deploy IPv6 Test Client + run: | + kubectl apply -f tests/manifests/test-client-ipv6.yaml + kubectl wait --for=condition=ready --timeout=300s pod/ipv6-test-client + + - name: Get NGF IPv6 Address + id: ngf-address + run: | + # Get the NGF service IPv6 address + NGF_IPV6=$(kubectl get service nginx-gateway -n nginx-gateway -o jsonpath='{.spec.clusterIP}') + echo "NGF IPv6 Address: $NGF_IPV6" + echo "ngf_ipv6=$NGF_IPV6" >> $GITHUB_OUTPUT + + - name: Run IPv6 Connectivity Tests + run: | + echo "=== Running IPv6-Only Tests ===" + + # Test 1: Basic connectivity test using test client pod + echo "Test 1: Basic IPv6 connectivity" + kubectl exec ipv6-test-client -- curl --version + kubectl exec ipv6-test-client -- nslookup nginx-gateway.nginx-gateway.svc.cluster.local + + # Test 2: Test NGF service directly via IPv6 + echo "Test 2: NGF Service IPv6 connectivity" + kubectl exec ipv6-test-client -- curl -6 --connect-timeout 30 --max-time 60 -v \ + -H "Host: ipv6-test.example.com" \ + "http://[${{ steps.ngf-address.outputs.ngf_ipv6 }}]:80/" || echo "Direct NGF test failed" + + # Test 3: Test via service DNS + echo "Test 3: Service DNS IPv6 connectivity" + kubectl exec ipv6-test-client -- curl -6 --connect-timeout 30 --max-time 60 -v \ + -H "Host: ipv6-test.example.com" \ + "http://nginx-gateway.nginx-gateway.svc.cluster.local:80/" || echo "Service DNS test failed" + + - name: Validate IPv6-Only Configuration + run: | + echo "=== Validating IPv6-Only Configuration ===" + + # Check NGF configuration + echo "NGF Pod IPv6 addresses:" + kubectl get pods -n nginx-gateway -o wide + + echo "NGF Service configuration:" + kubectl get service nginx-gateway -n nginx-gateway -o yaml + + echo "Gateway and HTTPRoute status:" + kubectl get gateway,httproute -A -o wide + + echo "Test application service configuration:" + kubectl get service test-app-ipv6-service -o yaml + + - name: Collect Logs + if: always() + run: | + echo "=== Collecting logs for debugging ===" + echo "NGF Controller logs:" + kubectl logs -n nginx-gateway deployment/nginx-gateway -c nginx-gateway-controller --tail=100 || true + + echo "NGINX logs:" + kubectl logs -n nginx-gateway deployment/nginx-gateway -c nginx --tail=100 || true + + echo "Test client logs:" + kubectl logs ipv6-test-client --tail=100 || true + + echo "Cluster events:" + kubectl get events --sort-by='.lastTimestamp' --all-namespaces --tail=50 || true + + - name: Cleanup + working-directory: ./tests + if: always() + run: | + bash scripts/cleanup-vm.sh true + bash scripts/cleanup-router.sh true + make delete-gke-cluster + rm -rf scripts/vars.env \ No newline at end of file diff --git a/.github/workflows/nfr.yml b/.github/workflows/nfr.yml index da4876fba6..deb8f933dc 100644 --- a/.github/workflows/nfr.yml +++ b/.github/workflows/nfr.yml @@ -3,20 +3,6 @@ name: Non Functional Testing on: workflow_dispatch: inputs: - test_label: - description: NFR test to run. Choose between a specific test or all tests - required: true - default: all - type: choice - options: - [ - performance, - upgrade, - scale, - zero-downtime-scale, - reconfiguration, - all, - ] version: description: Version of NGF under test required: true @@ -33,7 +19,6 @@ on: options: [oss, plus, both] schedule: - cron: "0 16 1,15 * *" # Run on the 1st and 15th of every month at 16:00 UTC - defaults: run: shell: bash @@ -41,10 +26,6 @@ defaults: env: PLUS_USAGE_ENDPOINT: ${{ secrets.JWT_PLUS_REPORTING_ENDPOINT }} -concurrency: - group: ${{ github.ref_name }}-nfr - cancel-in-progress: true - permissions: contents: read @@ -53,7 +34,6 @@ jobs: name: Set up vars runs-on: ubuntu-24.04 outputs: - test_label: ${{ github.event.inputs.test_label || 'all' }} version: ${{ github.event.inputs.version || 'edge' }} image_tag: ${{ github.event.inputs.image_tag || 'edge' }} types: ${{ steps.var.outputs.types }} @@ -70,7 +50,7 @@ jobs: fi setup-and-run-tests: - name: Setup and Run NFR Tests + name: Run IPv6-Only Tests runs-on: ubuntu-24.04 permissions: contents: read @@ -80,10 +60,13 @@ jobs: fail-fast: false matrix: type: ${{ fromJson(needs.vars.outputs.types) }} + env: + DOCKER_BUILD_SUMMARY: false steps: - name: Checkout Repository uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + ### Authenticate to GCP and set up gcloud, kubectl, and Docker - name: Authenticate to Google Cloud id: auth uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0 @@ -104,6 +87,7 @@ jobs: with: project_id: ${{ secrets.GCP_PROJECT_ID }} install_components: kubectl + ### - name: Setup dotenv file working-directory: ./tests/scripts @@ -116,6 +100,8 @@ jobs: echo "GKE_CLUSTER_NAME=nfr-tests-${{ github.run_id }}-${{ matrix.type }}" >> vars.env echo "GKE_CLUSTER_ZONE=us-west1-b" >> vars.env echo "GKE_CLUSTER_REGION=us-west1" >> vars.env + echo "GKE_CLUSTER_STACK_TYPE=ipv4-ipv6" >> vars.env + echo "GKE_CLUSTER_IPV6_ACCESS_TYPE=external" >> vars.env echo "GKE_PROJECT=${{ secrets.GCP_PROJECT_ID }}" >> vars.env echo "GKE_SVC_ACCOUNT=${{ secrets.GCP_SERVICE_ACCOUNT }}" >> vars.env echo "GKE_NODES_SERVICE_ACCOUNT=${{ secrets.GKE_NODES_SERVICE_ACCOUNT }}" >> vars.env @@ -126,10 +112,10 @@ jobs: echo "PLUS_ENABLED=${{ matrix.type == 'plus' }}" >> vars.env echo "GINKGO_LABEL=" >> vars.env echo "NGF_VERSION=${{ needs.vars.outputs.version }}" >> vars.env - echo "GKE_NUM_NODES=12" >> vars.env + echo "GKE_NUM_NODES=1" >> vars.env echo "GKE_MACHINE_TYPE=n2d-standard-16" >> vars.env + echo "IPV6_ENABLED=true" >> vars.env echo "PLUS_USAGE_ENDPOINT=${{ secrets.JWT_PLUS_REPORTING_ENDPOINT }}" >> vars.env - - name: Setup license file for plus if: matrix.type == 'plus' env: @@ -140,28 +126,52 @@ jobs: working-directory: ./tests run: make create-gke-cluster CI=true - - name: Create and setup VM - working-directory: ./tests - run: make create-and-setup-vm - - - name: Create and setup Router - working-directory: ./tests - run: make create-gke-router || true - - - name: Run Tests - working-directory: ./tests - run: | - if ${{ needs.vars.outputs.test_label != 'all' }}; then - sed -i '/^GINKGO_LABEL=/s/=.*/="${{ needs.vars.outputs.test_label }}"/' "scripts/vars.env" && make nfr-test CI=true; - else - make nfr-test CI=true; - fi - - - name: Upload Artifacts - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: results-${{ matrix.type }} - path: tests/results/**/*-${{ matrix.type }}.* + # - name: Create and setup VM + # working-directory: ./tests + # run: make create-and-setup-vm + + # - name: Create and setup Router + # working-directory: ./tests + # run: make create-gke-router || true + + # - name: Setup Golang Environment + # uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 + # with: + # go-version: stable + + # - name: Set GOPATH + # run: echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV + + # - name: NGF Docker meta + # id: ngf-meta + # uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0 + # with: + # images: | + # name=ghcr.io/nginx/nginx-gateway-fabric + # tags: | + # type=semver,pattern={{version}} + # type=schedule + # type=edge + # type=ref,event=pr + # type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} + + # - name: NGINX Docker meta + # id: nginx-meta + # uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0 + # with: + # images: | + # name=ghcr.io/nginx/nginx-gateway-fabric/${{ inputs.image_tag == 'plus' && 'nginx-plus' || inputs.image_tag }} + # tags: | + # type=semver,pattern={{version}} + # type=edge + # type=schedule + # type=ref,event=pr + # type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} + + # - name: Install and test NGF with IPv6 Configuration + # run: | + # ./scripts/run-tests-gcp-vm.sh + # working-directory: ./tests - name: Cleanup working-directory: ./tests @@ -171,40 +181,3 @@ jobs: bash scripts/cleanup-router.sh true make delete-gke-cluster rm -rf scripts/vars.env - - pr-results: - name: Open PR with results - runs-on: ubuntu-24.04 - permissions: - contents: write # needed for opening PR with the results files - pull-requests: write # needed for opening PR with the results files - needs: [vars, setup-and-run-tests] - steps: - - name: Checkout Repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - - name: Download Artifacts - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 - with: - path: tests/results/ - merge-multiple: true - - - name: Open a PR with the results - uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8 - with: - token: ${{ secrets.GITHUB_TOKEN }} - commit-message: NFR Test Results for NGF version ${{ needs.vars.outputs.version }} - author: nginx-bot - committer: nginx-bot - branch: tests/nfr-tests-${{ needs.vars.outputs.version }} - delete-branch: true - title: NFR Test Results for NGF version ${{ needs.vars.outputs.version }} - add-paths: | - tests/results/ - body: | - Update with NFR test results for NGF version ${{ needs.vars.outputs.version }} ${{ needs.vars.outputs.types }} - - Auto-generated by the NFR tests workflow run ${{ github.run_id }} - - Tests ran using Docker image tag ${{ needs.vars.outputs.image_tag }} - - ${{ needs.vars.outputs.test_label }} test(s) ran - assignees: ${{ github.actor }} - draft: ${{ github.event_name != 'schedule' }} diff --git a/.goreleaser.yml b/.goreleaser.yml index 31d20c536c..d6bcb68432 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -1,4 +1,5 @@ version: 2 +project_name: nginx-gateway-fabric env: - CGO_ENABLED=0 diff --git a/config/cluster/kind-ipv6-only.yaml b/config/cluster/kind-ipv6-only.yaml new file mode 100644 index 0000000000..bb2dce8392 --- /dev/null +++ b/config/cluster/kind-ipv6-only.yaml @@ -0,0 +1,7 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane +networking: + ipFamily: ipv6 + apiServerAddress: "::1" diff --git a/docs/developer/release-process.md b/docs/developer/release-process.md index 7cd056c9a3..d882e2419a 100644 --- a/docs/developer/release-process.md +++ b/docs/developer/release-process.md @@ -44,6 +44,8 @@ To create a new release, follow these steps: 4. Once the release branch pipeline completes, run tests using the `release-X.X-rc` images that are pushed to Github (for example, `release-1.3-rc`). 1. Kick off the [longevity tests](https://github.com/nginx/nginx-gateway-fabric/blob/main/tests/README.md#longevity-testing) for both OSS and Plus. You'll need to create two clusters and VMs for this. Before running, update your `vars.env` file with the proper image tag and prefixes. NGF and nginx images will be available from `ghcr.io`, and nginx plus will be available in GCP (`us-docker.pkg.dev//nginx-gateway-fabric/nginx-plus`). These tests need to run for 4 days before releasing. The results should be committed to the main branch and then cherry-picked to the release branch. 2. Kick off the [NFR workflow](https://github.com/nginx/nginx-gateway-fabric/actions/workflows/nfr.yml) in the browser. For `image_tag`, use `release-X.X-rc`, and for `version`, use the upcoming `X.Y.Z` NGF version. Run the workflow on the new release branch. This will run all of the NFR tests which are automated and open a PR with the results files when it is complete. Review this PR and make any necessary changes before merging. Once merged, be sure to cherry-pick the commit to the main branch as well (the original PR targets the release branch). + 3. Run the IPv6 tests using the `make ipv6-tests` target. This must be run from within the `tests` directory. This script need two arguments. The release version (e.g. `v2.1.0`) and the release image tag (e.g. `release-X.X-rc`). + For example, when running this script for release 2.1.0, it would look like this: `make ipv6-test RELEASE=2.1.1 RELEASE_IMAGE=release-2.1-rc` 5. Run the [Release PR](https://github.com/nginx/nginx-gateway-fabric/actions/workflows/release-pr.yml) workflow to update the repo files for the release. Then there are a few manual steps to complete: 1. Update the [README](/README.md) to include information about the release. 2. Update the [changelog](/CHANGELOG.md). There is going to be a new blank section generated by the automation that needs to be adjusted accordingly. diff --git a/tests/Makefile b/tests/Makefile index 668fd879e3..355dd40420 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -18,6 +18,8 @@ EXPERIMENTAL_CONFORMANCE_PROFILES = GATEWAY-TLS CONFORMANCE_PROFILES = $(STANDARD_CONFORMANCE_PROFILES) # by default we use the standard conformance profiles. If experimental is enabled we override this and add the experimental profiles. SKIP_TESTS = CEL_TEST_TARGET = +RELEASE = ? main # e.g. v2.1.0 +RELEASE_IMAGE = ? latest # e.g. release-2.1.0-rc # Check if ENABLE_EXPERIMENTAL is true ifeq ($(ENABLE_EXPERIMENTAL),true) @@ -118,6 +120,10 @@ start-longevity-test: nfr-test ## Start the longevity test to run for 4 days in stop-longevity-test: export STOP_LONGEVITY=true stop-longevity-test: nfr-test ## Stop the longevity test and collects results +.PHONY: ipv6-tests +ipv6-tests: GOARCH=amd64 +ipv6-tests: ## Example usage: make ipv6-tests RELEASE=vX.Y.Z RELEASE_IMAGE=release-X.Y-rc + ./ipv6/run-ipv6-test.sh $(RELEASE) $(RELEASE_IMAGE) .PHONY: .vm-nfr-test .vm-nfr-test: ## Runs the NFR tests on the GCP VM (called by `nfr-test`) diff --git a/tests/ipv6/config/kind-ipv6-only.yaml b/tests/ipv6/config/kind-ipv6-only.yaml new file mode 100644 index 0000000000..ab17d5f307 --- /dev/null +++ b/tests/ipv6/config/kind-ipv6-only.yaml @@ -0,0 +1,8 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: + ipFamily: ipv6 # Explicitly set the cluster to use IPv6 + apiServerAddress: "::1" + disableDefaultCNI: false # Use Kind's default CNI +nodes: +- role: control-plane diff --git a/tests/ipv6/manifests/gateway.yaml b/tests/ipv6/manifests/gateway.yaml new file mode 100644 index 0000000000..e6507f613b --- /dev/null +++ b/tests/ipv6/manifests/gateway.yaml @@ -0,0 +1,11 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: gateway +spec: + gatewayClassName: nginx + listeners: + - name: http + port: 80 + protocol: HTTP + hostname: "*.example.com" diff --git a/tests/ipv6/manifests/ipv6-test-app.yaml b/tests/ipv6/manifests/ipv6-test-app.yaml new file mode 100644 index 0000000000..202759eb23 --- /dev/null +++ b/tests/ipv6/manifests/ipv6-test-app.yaml @@ -0,0 +1,62 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-app-ipv6 + namespace: default +spec: + replicas: 1 + selector: + matchLabels: + app: test-app-ipv6 + template: + metadata: + labels: + app: test-app-ipv6 + spec: + containers: + - name: nginx + image: nginx:alpine + ports: + - containerPort: 80 + resources: + limits: + cpu: "100m" + memory: "128Mi" + requests: + cpu: "50m" + memory: "64Mi" +--- +apiVersion: v1 +kind: Service +metadata: + name: test-app-ipv6-service + namespace: default +spec: + selector: + app: test-app-ipv6 + ports: + - port: 80 + targetPort: 80 + ipFamilies: [IPv6] + ipFamilyPolicy: SingleStack +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: test-route-ipv6 + namespace: default +spec: + parentRefs: + - name: gateway + sectionName: http + namespace: default + hostnames: + - "ipv6-test.example.com" + rules: + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - name: test-app-ipv6-service + port: 80 diff --git a/tests/ipv6/manifests/ipv6-test-client.yaml b/tests/ipv6/manifests/ipv6-test-client.yaml new file mode 100644 index 0000000000..0eb7313742 --- /dev/null +++ b/tests/ipv6/manifests/ipv6-test-client.yaml @@ -0,0 +1,33 @@ +apiVersion: v1 +kind: Pod +metadata: + name: ipv6-test-client + namespace: default + labels: + app: ipv6-test-client +spec: + restartPolicy: Never + containers: + - name: test-client + image: curlimages/curl:8.11.1 + imagePullPolicy: IfNotPresent + command: ["sleep", "3600"] # Keep pod alive for exec commands + resources: + limits: + cpu: "100m" + memory: "128Mi" + requests: + cpu: "50m" + memory: "64Mi" + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 65534 + capabilities: + drop: + - ALL + dnsConfig: + options: + - name: single-request-reopen + - name: ndots + value: "2" diff --git a/tests/ipv6/run-ipv6-test.sh b/tests/ipv6/run-ipv6-test.sh new file mode 100755 index 0000000000..633727b228 --- /dev/null +++ b/tests/ipv6/run-ipv6-test.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash + +set -e # Exit immediately if a command exits with a non-zero status + +RELEASE=$1 +RELEASE_IMAGE=$2 + +if [[ -z "$RELEASE" || -z "$RELEASE_IMAGE" ]]; then + echo "Usage: $0 [HELM_RELEASE_NAME] [NAMESPACE] [CLUSTER_NAME]" + echo "Error: RELEASE and RELEASE_IMAGE are required parameters. Example usage `make ipv6-test RELEASE=vX.Y.Z RELEASE_IMAGE=release-X.Y-rc`" + exit 1 +fi + +HELM_RELEASE_NAME=${3:-ngf} +NAMESPACE=${4:-nginx-gateway} +CLUSTER_NAME=${5:-ipv6-only-${RELEASE}} +RELEASE_REPO=ghcr.io/nginx/nginx-gateway-fabric + +cleanup() { + echo "Cleaning up resources..." + kubectl delete -f ipv6/manifests/ipv6-test-app.yaml || true + kubectl delete -f ipv6/manifests/ipv6-test-client.yaml || true + kubectl delete -f ipv6/manifests/gateway.yaml || true + helm uninstall ${HELM_RELEASE_NAME} -n ${NAMESPACE} || true + kind delete cluster --name ${CLUSTER_NAME} || true +} + +trap cleanup EXIT + +echo "Creating IPv6 kind cluster..." +kind create cluster --name ${CLUSTER_NAME} --config ipv6/config/kind-ipv6-only.yaml + +echo "Applying Gateway API CRDs" +kubectl kustomize "https://github.com/nginx/nginx-gateway-fabric/config/crd/gateway-api/standard?ref=${RELEASE}" | kubectl apply -f - + +echo "Applying NGF CRDs" +kubectl apply --server-side -f https://raw.githubusercontent.com/nginx/nginx-gateway-fabric/${RELEASE}/deploy/crds.yaml + +echo "Pulling NGF image ${RELEASE_REPO}..." +docker pull ${RELEASE_REPO}:${RELEASE_IMAGE} + +echo "Loading NGF image into kind cluster..." +docker save ${RELEASE_REPO}:${RELEASE_IMAGE} | docker exec -i ${CLUSTER_NAME}-control-plane ctr --namespace=k8s.io images import - + +helm upgrade --install ${HELM_RELEASE_NAME} oci://ghcr.io/nginx/charts/nginx-gateway-fabric \ + --create-namespace -n ${NAMESPACE} \ + --set nginx.config.ipFamily=ipv6 \ + --set nginx.service.type=ClusterIP \ + --set nginxGateway.image.repository=${RELEASE_REPO} \ + --set nginxGateway.image.tag=${RELEASE_IMAGE} + +echo "Deploying Gateway..." +kubectl apply -f ipv6/manifests/gateway.yaml +echo "Waiting for NGINX Gateway to be ready..." +kubectl wait --for=condition=accepted --timeout=300s gateway/gateway +POD_NAME=$(kubectl get pods -l app.kubernetes.io/instance=${HELM_RELEASE_NAME} -o jsonpath='{.items[0].metadata.name}') +kubectl wait --for=condition=ready --timeout=300s pod/${POD_NAME} + +echo "Deploying IPv6 test application" +kubectl apply -f ipv6/manifests/ipv6-test-app.yaml + +echo "Waiting for NGF to be ready..." +kubectl wait --for=condition=available --timeout=300s deployment/${HELM_RELEASE_NAME}-nginx-gateway-fabric -n ${NAMESPACE} + +echo "Waiting for test applications to be ready..." +kubectl wait --for=condition=available --timeout=300s deployment/test-app-ipv6 + +echo "Deploying IPv6 test client" +kubectl apply -f ipv6/manifests/ipv6-test-client.yaml +kubectl wait --for=condition=ready --timeout=300s pod/ipv6-test-client + +echo "Getting NGF service IPv6 address" +NGF_IPV6=$(kubectl get service gateway-nginx -o jsonpath='{.spec.clusterIP}') +echo "NGF IPv6 Address: $NGF_IPV6" + +echo "=== Running IPv6-Only Tests ===" + +echo "== Test 1: Basic IPv6 connectivity ==" +kubectl exec ipv6-test-client -- curl --version +kubectl exec ipv6-test-client -- nslookup gateway-nginx.default.svc.cluster.local || echo "Test 1: Basic IPv6 connectivity failed" +test1_status=$? + +if [[ $test1_status -eq 0 ]]; then + echo "✅ Test 1: Basic IPv6 connectivity succeeded" +fi + +echo "== Test 2: NGF Service IPv6 connectivity ==" +kubectl exec ipv6-test-client -- curl -6 --connect-timeout 30 --max-time 60 -v \ + -H "Host: ipv6-test.example.com" \ + "http://[${NGF_IPV6}]:80/" || echo "Test 2: NGF Service IPv6 connectivity failed" +test2_status=$? + +if [[ $test2_status -eq 0 ]]; then + echo "✅ Test 2: NGF Service IPv6 connectivity succeeded" +fi + +echo "== Test 3: Service DNS IPv6 connectivity ==" +kubectl exec ipv6-test-client -- curl -6 --connect-timeout 30 --max-time 60 -v \ + -H "Host: ipv6-test.example.com" \ + "http://gateway-nginx.default.svc.cluster.local:80/" || echo "Test 3: Service DNS IPv6 connectivity failed" +test3_status=$? + +if [[ $test3_status -eq 0 ]]; then + echo "✅ Test 3: Service DNS IPv6 connectivity succeeded" +fi + +if [[ $test1_status -eq 0 && $test2_status -eq 0 && $test3_status -eq 0 ]]; then + echo -e "✅ All tests passed!" +else + echo -e "\033[31m One or more tests failed. \033[0m" +fi \ No newline at end of file diff --git a/tests/scripts/cleanup-router.sh b/tests/scripts/cleanup-router.sh index ee3ea524b2..41152397e7 100755 --- a/tests/scripts/cleanup-router.sh +++ b/tests/scripts/cleanup-router.sh @@ -4,5 +4,5 @@ set -o pipefail source scripts/vars.env -gcloud compute routers nats delete "${RESOURCE_NAME}" --quiet --router "${RESOURCE_NAME}" --router-region "${GKE_CLUSTER_REGION}" -gcloud compute routers delete "${RESOURCE_NAME}" --quiet --region "${GKE_CLUSTER_REGION}" +gcloud compute routers nats delete "${RESOURCE_NAME}" --quiet --router "${RESOURCE_NAME}" --router-region "${GKE_CLUSTER_REGION}" || true +gcloud compute routers delete "${RESOURCE_NAME}" --quiet --region "${GKE_CLUSTER_REGION}" || true diff --git a/tests/scripts/cleanup-vm.sh b/tests/scripts/cleanup-vm.sh index ecb0420d18..59b3eae661 100755 --- a/tests/scripts/cleanup-vm.sh +++ b/tests/scripts/cleanup-vm.sh @@ -13,5 +13,12 @@ if [ "${ADD_VM_IP_AUTH_NETWORKS}" = "true" ] && [ "${skip_gke_master_control_nod gcloud container clusters update "${GKE_CLUSTER_NAME}" --zone "${GKE_CLUSTER_ZONE}" --enable-master-authorized-networks --master-authorized-networks="${CURRENT_AUTH_NETWORK}" fi -gcloud compute instances delete "${RESOURCE_NAME}" --quiet --project="${GKE_PROJECT}" --zone="${GKE_CLUSTER_ZONE}" -gcloud compute firewall-rules delete "${RESOURCE_NAME}" --quiet --project="${GKE_PROJECT}" +gcloud compute instances delete "${RESOURCE_NAME}" --quiet --project="${GKE_PROJECT}" --zone="${GKE_CLUSTER_ZONE}" || true + +gcloud compute firewall-rules delete "${RESOURCE_NAME}" --quiet --project="${GKE_PROJECT}" || true + +# Clean up the custom network and subnet if IPv6 was enabled +# if [ "${IPV6_ENABLED}" = "true" ]; then +# gcloud compute networks subnets delete ${RESOURCE_NAME} --region=${GKE_CLUSTER_REGION} --quiet || true +# gcloud compute networks delete ${RESOURCE_NAME} --quiet || true +# fi diff --git a/tests/scripts/create-and-setup-gcp-vm.sh b/tests/scripts/create-and-setup-gcp-vm.sh index 5031e2e9ac..c799142fc4 100755 --- a/tests/scripts/create-and-setup-gcp-vm.sh +++ b/tests/scripts/create-and-setup-gcp-vm.sh @@ -5,30 +5,69 @@ set -o pipefail SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) REPO_DIR=$(dirname $(dirname "$SCRIPT_DIR")) +# Default network settings +NETWORK=default +STACK_TYPE="IPV4_ONLY" +NETWORK_TIER="network-tier=PREMIUM" + source scripts/vars.env +# Create custom network and subnet if IPv6 is enabled +# For IPv6, we create a dual-stack subnet with internal IPv6 addresses and external IPv4 addresses +if [ "${IPV6_ENABLED}" = "true" ]; then + echo "Using IPv6 Network interface for the GKE cluster" + # gcloud compute networks create ${RESOURCE_NAME} --subnet-mode=custom --enable-ula-internal-ipv6 --quiet # The --subnet-mode=custom flag allows us to create custom subnets + + # gcloud compute networks subnets create ${RESOURCE_NAME} \ + # --network=${RESOURCE_NAME} \ + # --stack-type=IPV4_IPV6 \ + # --ipv6-access-type=INTERNAL \ + # --region=${GKE_CLUSTER_REGION} \ + # --range=10.120.0.0/14 + + # Dual-Stack Settings + # NETWORK="us-ipv4-ipv6" + # STACK_TYPE="IPV4_IPV6" + + # IPv6 Only Settings + NETWORK="ipv6-only" + NETWORK_TIER="ipv6-network-tier=PREMIUM" # This will work only if STACK_TYPE is IPV6_ONLY + STACK_TYPE="IPV6_ONLY" +fi + gcloud compute firewall-rules create "${RESOURCE_NAME}" \ --project="${GKE_PROJECT}" \ --direction=INGRESS \ --priority=1000 \ - --network=default \ + --network=${NETWORK} \ --action=ALLOW \ --rules=tcp:22 \ --source-ranges="${SOURCE_IP_RANGE}" \ --target-tags="${NETWORK_TAGS}" gcloud compute instances create "${RESOURCE_NAME}" --project="${GKE_PROJECT}" --zone="${GKE_CLUSTER_ZONE}" --machine-type=n2-standard-2 \ - --network-interface=network-tier=PREMIUM,stack-type=IPV4_ONLY,subnet=default --maintenance-policy=MIGRATE \ + --network-interface=${NETWORK_TIER},stack-type=${STACK_TYPE},subnet=${NETWORK} --maintenance-policy=MIGRATE \ --provisioning-model=STANDARD --service-account="${GKE_SVC_ACCOUNT}" \ --scopes=https://www.googleapis.com/auth/devstorage.read_only,https://www.googleapis.com/auth/logging.write,https://www.googleapis.com/auth/monitoring.write,https://www.googleapis.com/auth/servicecontrol,https://www.googleapis.com/auth/service.management.readonly,https://www.googleapis.com/auth/trace.append,https://www.googleapis.com/auth/cloud-platform \ --tags="${NETWORK_TAGS}" --create-disk=auto-delete=yes,boot=yes,device-name="${RESOURCE_NAME}",image-family=projects/"${GKE_PROJECT}"/global/images/ngf-debian,mode=rw,size=20 --no-shielded-secure-boot --shielded-vtpm --shielded-integrity-monitoring --labels=goog-ec-src=vm_add-gcloud --reservation-affinity=any # Add VM IP to GKE master control node access, if required if [ "${ADD_VM_IP_AUTH_NETWORKS}" = "true" ]; then + + # EXTERNAL_IP=$(gcloud compute instances describe "${RESOURCE_NAME}" --project="${GKE_PROJECT}" --zone="${GKE_CLUSTER_ZONE}" \ + # --format='value(networkInterfaces[0].accessConfigs[0].natIP)') + + echo "IPv6 is enabled, fetching the external IPv6 address" EXTERNAL_IP=$(gcloud compute instances describe "${RESOURCE_NAME}" --project="${GKE_PROJECT}" --zone="${GKE_CLUSTER_ZONE}" \ - --format='value(networkInterfaces[0].accessConfigs[0].natIP)') + --format='value(networkInterfaces[0].ipv6AccessConfigs[0].externalIpv6)') + + echo "External IP of the VM is: ${EXTERNAL_IP}" + CURRENT_AUTH_NETWORK=$(gcloud container clusters describe "${GKE_CLUSTER_NAME}" --zone="${GKE_CLUSTER_ZONE}" \ - --format="value(masterAuthorizedNetworksConfig.cidrBlocks[0])" | sed 's/cidrBlock=//') + --format="value(masterAuthorizedNetworksConfig.cidrBlocks[0])" | sed 's/cidrBlock=//') + + echo "Current GKE master authorized networks: ${CURRENT_AUTH_NETWORK}" + gcloud container clusters update "${GKE_CLUSTER_NAME}" --zone="${GKE_CLUSTER_ZONE}" --enable-master-authorized-networks --master-authorized-networks="${EXTERNAL_IP}"/32,"${CURRENT_AUTH_NETWORK}" fi diff --git a/tests/scripts/create-gke-cluster.sh b/tests/scripts/create-gke-cluster.sh index f0ccbfcc2b..b049effb3e 100755 --- a/tests/scripts/create-gke-cluster.sh +++ b/tests/scripts/create-gke-cluster.sh @@ -31,7 +31,12 @@ gcloud container clusters create "${GKE_CLUSTER_NAME}" \ --logging=SYSTEM,WORKLOAD \ --machine-type "${GKE_MACHINE_TYPE}" \ --num-nodes "${GKE_NUM_NODES}" \ - --no-enable-insecure-kubelet-readonly-port + --no-enable-insecure-kubelet-readonly-port \ + --stack-type="${GKE_CLUSTER_STACK_TYPE}" \ + --ipv6-access-type="${GKE_CLUSTER_IPV6_ACCESS_TYPE}" \ + --create-subnetwork name=${RESOURCE_NAME},range=2600:1900:4041:625:0:1:0:0/96 \ + --network=us-ipv4-ipv6 --enable-dataplane-v2 + # Add current IP to GKE master control node access, if this script is not invoked during a CI run. if [ "${IS_CI}" = "false" ]; then diff --git a/tests/scripts/remote-scripts/run-nfr-tests.sh b/tests/scripts/remote-scripts/run-nfr-tests.sh index baea40ab52..eb941ed494 100755 --- a/tests/scripts/remote-scripts/run-nfr-tests.sh +++ b/tests/scripts/remote-scripts/run-nfr-tests.sh @@ -10,7 +10,7 @@ elif [ "${STOP_LONGEVITY}" == "true" ]; then GINKGO_LABEL="longevity-teardown" fi -cd nginx-gateway-fabric/tests && make .vm-nfr-test CI=${CI} TAG="${TAG}" PREFIX="${PREFIX}" NGINX_PREFIX="${NGINX_PREFIX}" NGINX_PLUS_PREFIX="${NGINX_PLUS_PREFIX}" PLUS_ENABLED="${PLUS_ENABLED}" GINKGO_LABEL=${GINKGO_LABEL} GINKGO_FLAGS="${GINKGO_FLAGS}" PULL_POLICY=Always GW_SERVICE_TYPE=LoadBalancer NGF_VERSION="${NGF_VERSION}" PLUS_USAGE_ENDPOINT="${PLUS_USAGE_ENDPOINT}" GKE_PROJECT="${GKE_PROJECT}" +cd nginx-gateway-fabric/tests && make .vm-nfr-test CI=${CI} TAG="${TAG}" PREFIX="${PREFIX}" NGINX_PREFIX="${NGINX_PREFIX}" NGINX_PLUS_PREFIX="${NGINX_PLUS_PREFIX}" PLUS_ENABLED="${PLUS_ENABLED}" GINKGO_LABEL=${GINKGO_LABEL} GINKGO_FLAGS="${GINKGO_FLAGS}" PULL_POLICY=Always GW_SERVICE_TYPE=LoadBalancer NGF_VERSION="${NGF_VERSION}" PLUS_USAGE_ENDPOINT="${PLUS_USAGE_ENDPOINT}" GKE_PROJECT="${GKE_PROJECT}" HELM_PARAMETERS="${HELM_PARAMETERS}" if [ "${START_LONGEVITY}" == "true" ]; then suite/scripts/longevity-wrk.sh diff --git a/tests/scripts/run-tests-gcp-vm.sh b/tests/scripts/run-tests-gcp-vm.sh index 3cff298694..4254910fa5 100755 --- a/tests/scripts/run-tests-gcp-vm.sh +++ b/tests/scripts/run-tests-gcp-vm.sh @@ -8,51 +8,62 @@ source scripts/vars.env gcloud compute scp --zone "${GKE_CLUSTER_ZONE}" --project="${GKE_PROJECT}" "${SCRIPT_DIR}"/vars.env username@"${RESOURCE_NAME}":~ -gcloud compute ssh --zone "${GKE_CLUSTER_ZONE}" --project="${GKE_PROJECT}" username@"${RESOURCE_NAME}" \ - --command="export START_LONGEVITY=${START_LONGEVITY} &&\ - export STOP_LONGEVITY=${STOP_LONGEVITY} &&\ - export CI=${CI} &&\ - bash -s" <"${SCRIPT_DIR}"/remote-scripts/run-nfr-tests.sh -retcode=$? - -if [ ${retcode} -ne 0 ]; then - echo "Error running tests on VM" - exit 1 -fi +if [ "${IPV6_ENABLED}" = "true" ]; then + gcloud compute ssh --zone "${GKE_CLUSTER_ZONE}" --project="${GKE_PROJECT}" username@"${RESOURCE_NAME}" \ + --command="bash -s" <"${SCRIPT_DIR}"/remote-scripts/run-ipv6-test.sh + retcode=$? -## Use rsync if running locally (faster); otherwise if in the pipeline don't download an SSH config -if [ "${CI}" = "false" ]; then - gcloud compute config-ssh --ssh-config-file ngf-gcp.ssh >/dev/null - rsync -ave 'ssh -F ngf-gcp.ssh' username@"${RESOURCE_NAME}"."${GKE_CLUSTER_ZONE}"."${GKE_PROJECT}":~/nginx-gateway-fabric/tests/results . + if [ ${retcode} -ne 0 ]; then + echo "Error running IPv6 tests on VM" + exit 1 + fi else - gcloud compute scp --zone "${GKE_CLUSTER_ZONE}" --project="${GKE_PROJECT}" --recurse username@"${RESOURCE_NAME}":~/nginx-gateway-fabric/tests/results . -fi + gcloud compute ssh --zone "${GKE_CLUSTER_ZONE}" --project="${GKE_PROJECT}" username@"${RESOURCE_NAME}" \ + --command="export START_LONGEVITY=${START_LONGEVITY} &&\ + export STOP_LONGEVITY=${STOP_LONGEVITY} &&\ + export CI=${CI} &&\ + bash -s" <"${SCRIPT_DIR}"/remote-scripts/run-nfr-tests.sh + retcode=$? -## If tearing down the longevity test, we need to collect logs from gcloud and add to the results -if [ "${STOP_LONGEVITY}" = "true" ]; then - version=${NGF_VERSION} - if [ "${version}" = "" ]; then - version=${TAG} + if [ ${retcode} -ne 0 ]; then + echo "Error running tests on VM" + exit 1 fi - runType=oss - if [ "${PLUS_ENABLED}" = "true" ]; then - runType=plus + ## Use rsync if running locally (faster); otherwise if in the pipeline don't download an SSH config + if [ "${CI}" = "false" ]; then + gcloud compute config-ssh --ssh-config-file ngf-gcp.ssh >/dev/null + rsync -ave 'ssh -F ngf-gcp.ssh' username@"${RESOURCE_NAME}"."${GKE_CLUSTER_ZONE}"."${GKE_PROJECT}":~/nginx-gateway-fabric/tests/results . + else + gcloud compute scp --zone "${GKE_CLUSTER_ZONE}" --project="${GKE_PROJECT}" --recurse username@"${RESOURCE_NAME}":~/nginx-gateway-fabric/tests/results . fi - results="${SCRIPT_DIR}/../results/longevity/$version/$version-$runType.md" - printf "\n## Error Logs\n\n" >>"${results}" + ## If tearing down the longevity test, we need to collect logs from gcloud and add to the results + if [ "${STOP_LONGEVITY}" = "true" ]; then + version=${NGF_VERSION} + if [ "${version}" = "" ]; then + version=${TAG} + fi + + runType=oss + if [ "${PLUS_ENABLED}" = "true" ]; then + runType=plus + fi + + results="${SCRIPT_DIR}/../results/longevity/$version/$version-$runType.md" + printf "\n## Error Logs\n\n" >>"${results}" - ## ngf error logs - ngfErrText=$(gcloud logging read --project="${GKE_PROJECT}" 'resource.labels.cluster_name='"${RESOURCE_NAME}"' AND resource.type=k8s_container AND resource.labels.container_name=nginx-gateway AND labels."k8s-pod/app_kubernetes_io/instance"=ngf-longevity AND severity=ERROR AND SEARCH("error")' --format "value(textPayload)") - ngfErrJSON=$(gcloud logging read --project="${GKE_PROJECT}" 'resource.labels.cluster_name='"${RESOURCE_NAME}"' AND resource.type=k8s_container AND resource.labels.container_name=nginx-gateway AND labels."k8s-pod/app_kubernetes_io/instance"=ngf-longevity AND severity=ERROR AND SEARCH("error")' --format "value(jsonPayload)") - printf "### nginx-gateway\n%s\n%s\n\n" "${ngfErrText}" "${ngfErrJSON}" >>"${results}" + ## ngf error logs + ngfErrText=$(gcloud logging read --project="${GKE_PROJECT}" 'resource.labels.cluster_name='"${RESOURCE_NAME}"' AND resource.type=k8s_container AND resource.labels.container_name=nginx-gateway AND labels."k8s-pod/app_kubernetes_io/instance"=ngf-longevity AND severity=ERROR AND SEARCH("error")' --format "value(textPayload)") + ngfErrJSON=$(gcloud logging read --project="${GKE_PROJECT}" 'resource.labels.cluster_name='"${RESOURCE_NAME}"' AND resource.type=k8s_container AND resource.labels.container_name=nginx-gateway AND labels."k8s-pod/app_kubernetes_io/instance"=ngf-longevity AND severity=ERROR AND SEARCH("error")' --format "value(jsonPayload)") + printf "### nginx-gateway\n%s\n%s\n\n" "${ngfErrText}" "${ngfErrJSON}" >>"${results}" - ## nginx error logs - ngxErr=$(gcloud logging read --project="${GKE_PROJECT}" 'resource.labels.cluster_name='"${RESOURCE_NAME}"' AND resource.type=k8s_container AND resource.labels.container_name=nginx AND labels."k8s-pod/app_kubernetes_io/instance"=ngf-longevity AND severity=ERROR AND SEARCH("`[warn]`") OR SEARCH("`[error]`") OR SEARCH("`[emerg]`")' --format "value(textPayload)") - printf "### nginx\n%s\n\n" "${ngxErr}" >>"${results}" + ## nginx error logs + ngxErr=$(gcloud logging read --project="${GKE_PROJECT}" 'resource.labels.cluster_name='"${RESOURCE_NAME}"' AND resource.type=k8s_container AND resource.labels.container_name=nginx AND labels."k8s-pod/app_kubernetes_io/instance"=ngf-longevity AND severity=ERROR AND SEARCH("`[warn]`") OR SEARCH("`[error]`") OR SEARCH("`[emerg]`")' --format "value(textPayload)") + printf "### nginx\n%s\n\n" "${ngxErr}" >>"${results}" - ## nginx non-200 responses (also filter out 499 since wrk cancels connections) - ngxNon200=$(gcloud logging read --project="${GKE_PROJECT}" 'resource.labels.cluster_name='"${RESOURCE_NAME}"' AND resource.type=k8s_container AND resource.labels.container_name=nginx AND labels."k8s-pod/app_kubernetes_io/instance"=ngf-longevity AND "GET" "HTTP/1.1" -"200" -"499" -"client prematurely closed connection"' --format "value(textPayload)") - printf "%s\n\n" "${ngxNon200}" >>"${results}" + ## nginx non-200 responses (also filter out 499 since wrk cancels connections) + ngxNon200=$(gcloud logging read --project="${GKE_PROJECT}" 'resource.labels.cluster_name='"${RESOURCE_NAME}"' AND resource.type=k8s_container AND resource.labels.container_name=nginx AND labels."k8s-pod/app_kubernetes_io/instance"=ngf-longevity AND "GET" "HTTP/1.1" -"200" -"499" -"client prematurely closed connection"' --format "value(textPayload)") + printf "%s\n\n" "${ngxNon200}" >>"${results}" + fi fi diff --git a/tests/suite/manifests/snippets-filter/invalid-duplicate-sf.yaml b/tests/suite/manifests/snippets-filter/invalid-duplicate-sf.yaml index ba7e5b6c55..137771ed4e 100644 --- a/tests/suite/manifests/snippets-filter/invalid-duplicate-sf.yaml +++ b/tests/suite/manifests/snippets-filter/invalid-duplicate-sf.yaml @@ -19,21 +19,21 @@ metadata: name: tea spec: parentRefs: - - name: gateway - sectionName: http + - name: gateway + sectionName: http hostnames: - - "cafe.example.com" + - "cafe.example.com" rules: - - matches: - - path: - type: Exact - value: /tea - filters: - - type: ExtensionRef - extensionRef: - group: gateway.nginx.org - kind: SnippetsFilter - name: duplicate-directive - backendRefs: - - name: tea - port: 80 + - matches: + - path: + type: Exact + value: /tea + filters: + - type: ExtensionRef + extensionRef: + group: gateway.nginx.org + kind: SnippetsFilter + name: duplicate-directive + backendRefs: + - name: tea + port: 80