Skip to content

v1.6.2 failed to start control loop: failed to prepare the first batch: no matches for kind "ObservabilityPolicy" in version "gateway.nginx.org/v1alpha2" #3340

@ghost

Description

Describe the bug
can't start nginx-fabric pod immediatly after installing with this error
failed to start control loop: failed to prepare the first batch: no matches for kind "ObservabilityPolicy" in version "gateway.nginx.org/v1alpha2"

$ kubectl logs -n nginx-gateway ngf-nginx-gateway-fabric-5c4bfb8c48-s574v
Defaulted container "nginx-gateway" out of: nginx-gateway, nginx, init (init) {"level":"info","ts":"2025-04-28T03:48:39Z","msg":"Starting NGINX Gateway Fabric in static mode","version":"1.6.2","commit":"532db6a20b2912fe397211eef9f8d564d46a4bdd","date":"2025-03-11T17:28:32Z","dirty":"false"} {"level":"info","ts":"2025-04-28T03:48:39Z","msg":"Starting manager"} {"level":"info","ts":"2025-04-28T03:48:39Z","logger":"controller-runtime.metrics","msg":"Starting metrics server"} {"level":"info","ts":"2025-04-28T03:48:39Z","logger":"controller-runtime.metrics","msg":"Serving metrics server","bindAddress":":9113","secure":false} {"level":"info","ts":"2025-04-28T03:48:39Z","msg":"starting server","name":"health probe","addr":"[::]:8081"} {"level":"info","ts":"2025-04-28T03:48:39Z","msg":"attempting to acquire leader lease nginx-gateway/ngf-nginx-gateway-fabric-leader-election..."} {"level":"error","ts":"2025-04-28T03:48:39Z","logger":"controller-runtime.source.EventHandler","msg":"if kind is a CRD, it should be installed before calling Start","kind":"ObservabilityPolicy.gateway.nginx.org","error":"no matches for kind \"ObservabilityPolicy\" in version \"gateway.nginx.org/v1alpha2\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/source.(*Kind[...]).Start.func1.1\n\tpkg/mod/sigs.k8s.io/[email protected]/pkg/internal/source/kind.go:71\nk8s.io/apimachinery/pkg/util/wait.loopConditionUntilContext.func1\n\tpkg/mod/k8s.io/[email protected]/pkg/util/wait/loop.go:53\nk8s.io/apimachinery/pkg/util/wait.loopConditionUntilContext\n\tpkg/mod/k8s.io/[email protected]/pkg/util/wait/loop.go:54\nk8s.io/apimachinery/pkg/util/wait.PollUntilContextCancel\n\tpkg/mod/k8s.io/[email protected]/pkg/util/wait/poll.go:33\nsigs.k8s.io/controller-runtime/pkg/internal/source.(*Kind[...]).Start.func1\n\tpkg/mod/sigs.k8s.io/[email protected]/pkg/internal/source/kind.go:64"} {"level":"error","ts":"2025-04-28T03:48:39Z","logger":"controller-runtime.source.EventHandler","msg":"failed to get informer from cache","error":"Timeout: failed waiting for *v1.Namespace Informer to sync","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/source.(*Kind[...]).Start.func1.1\n\tpkg/mod/sigs.k8s.io/[email protected]/pkg/internal/source/kind.go:76\nk8s.io/apimachinery/pkg/util/wait.loopConditionUntilContext.func1\n\tpkg/mod/k8s.io/[email protected]/pkg/util/wait/loop.go:53\nk8s.io/apimachinery/pkg/util/wait.loopConditionUntilContext\n\tpkg/mod/k8s.io/[email protected]/pkg/util/wait/loop.go:54\nk8s.io/apimachinery/pkg/util/wait.PollUntilContextCancel\n\tpkg/mod/k8s.io/[email protected]/pkg/util/wait/poll.go:33\nsigs.k8s.io/controller-runtime/pkg/internal/source.(*Kind[...]).Start.func1\n\tpkg/mod/sigs.k8s.io/[email protected]/pkg/internal/source/kind.go:64"} {"level":"error","ts":"2025-04-28T03:48:39Z","logger":"controller-runtime.source.EventHandler","msg":"failed to get informer from cache","error":"Timeout: failed waiting for *v1alpha1.ClientSettingsPolicy Informer to sync","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/source.(*Kind[...]).Start.func1.1\n\tpkg/mod/sigs.k8s.io/[email protected]/pkg/internal/source/kind.go:76\nk8s.io/apimachinery/pkg/util/wait.loopConditionUntilContext.func1\n\tpkg/mod/k8s.io/[email protected]/pkg/util/wait/loop.go:53\nk8s.io/apimachinery/pkg/util/wait.loopConditionUntilContext\n\tpkg/mod/k8s.io/[email protected]/pkg/util/wait/loop.go:54\nk8s.io/apimachinery/pkg/util/wait.PollUntilContextCancel\n\tpkg/mod/k8s.io/[email protected]/pkg/util/wait/poll.go:33\nsigs.k8s.io/controller-runtime/pkg/internal/source.(*Kind[...]).Start.func1\n\tpkg/mod/sigs.k8s.io/[email protected]/pkg/internal/source/kind.go:64"} {"level":"error","ts":"2025-04-28T03:48:39Z","logger":"controller-runtime.source.EventHandler","msg":"failed to get informer from cache","error":"Timeout: failed waiting for *v1.Secret Informer to sync","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/source.(*Kind[...]).Start.func1.1\n\tpkg/mod/sigs.k8s.io/[email protected]/pkg/internal/source/kind.go:76\nk8s.io/apimachinery/pkg/util/wait.loopConditionUntilContext.func1\n\tpkg/mod/k8s.io/[email protected]/pkg/util/wait/loop.go:53\nk8s.io/apimachinery/pkg/util/wait.loopConditionUntilContext\n\tpkg/mod/k8s.io/[email protected]/pkg/util/wait/loop.go:54\nk8s.io/apimachinery/pkg/util/wait.PollUntilContextCancel\n\tpkg/mod/k8s.io/[email protected]/pkg/util/wait/poll.go:33\nsigs.k8s.io/controller-runtime/pkg/internal/source.(*Kind[...]).Start.func1\n\tpkg/mod/sigs.k8s.io/[email protected]/pkg/internal/source/kind.go:64"} {"level":"error","ts":"2025-04-28T03:48:39Z","logger":"controller-runtime.source.EventHandler","msg":"failed to get informer from cache","error":"Timeout: failed waiting for *v1.HTTPRoute Informer to sync","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/source.(*Kind[...]).Start.func1.1\n\tpkg/mod/sigs.k8s.io/[email protected]/pkg/internal/source/kind.go:76\nk8s.io/apimachinery/pkg/util/wait.loopConditionUntilContext.func1\n\tpkg/mod/k8s.io/[email protected]/pkg/util/wait/loop.go:53\nk8s.io/apimachinery/pkg/util/wait.loopConditionUntilContext\n\tpkg/mod/k8s.io/[email protected]/pkg/util/wait/loop.go:54\nk8s.io/apimachinery/pkg/util/wait.PollUntilContextCancel\n\tpkg/mod/k8s.io/[email protected]/pkg/util/wait/poll.go:33\nsigs.k8s.io/controller-runtime/pkg/internal/source.(*Kind[...]).Start.func1\n\tpkg/mod/sigs.k8s.io/[email protected]/pkg/internal/source/kind.go:64"} {"level":"error","ts":"2025-04-28T03:48:39Z","logger":"controller-runtime.source.EventHandler","msg":"failed to get informer from cache","error":"Timeout: failed waiting for *v1.GRPCRoute Informer to sync","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/source.(*Kind[...]).Start.func1.1\n\tpkg/mod/sigs.k8s.io/[email protected]/pkg/internal/source/kind.go:76\nk8s.io/apimachinery/pkg/util/wait.loopConditionUntilContext.func1\n\tpkg/mod/k8s.io/[email protected]/pkg/util/wait/loop.go:53\nk8s.io/apimachinery/pkg/util/wait.loopConditionUntilContext\n\tpkg/mod/k8s.io/[email protected]/pkg/util/wait/loop.go:54\nk8s.io/apimachinery/pkg/util/wait.PollUntilContextCancel\n\tpkg/mod/k8s.io/[email protected]/pkg/util/wait/poll.go:33\nsigs.k8s.io/controller-runtime/pkg/internal/source.(*Kind[...]).Start.func1\n\tpkg/mod/sigs.k8s.io/[email protected]/pkg/internal/source/kind.go:64"} {"level":"error","ts":"2025-04-28T03:48:39Z","logger":"controller-runtime.source.EventHandler","msg":"failed to get informer from cache","error":"Timeout: failed waiting for *v1.PartialObjectMetadata Informer to sync","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/source.(*Kind[...]).Start.func1.1\n\tpkg/mod/sigs.k8s.io/[email protected]/pkg/internal/source/kind.go:76\nk8s.io/apimachinery/pkg/util/wait.loopConditionUntilContext.func1\n\tpkg/mod/k8s.io/[email protected]/pkg/util/wait/loop.go:53\nk8s.io/apimachinery/pkg/util/wait.loopConditionUntilContext\n\tpkg/mod/k8s.io/[email protected]/pkg/util/wait/loop.go:54\nk8s.io/apimachinery/pkg/util/wait.PollUntilContextCancel\n\tpkg/mod/k8s.io/[email protected]/pkg/util/wait/poll.go:33\nsigs.k8s.io/controller-runtime/pkg/internal/source.(*Kind[...]).Start.func1\n\tpkg/mod/sigs.k8s.io/[email protected]/pkg/internal/source/kind.go:64"} {"level":"error","ts":"2025-04-28T03:48:39Z","logger":"controller-runtime.source.EventHandler","msg":"failed to get informer from cache","error":"Timeout: failed waiting for *v1alpha1.NginxGateway Informer to sync","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/source.(*Kind[...]).Start.func1.1\n\tpkg/mod/sigs.k8s.io/[email protected]/pkg/internal/source/kind.go:76\nk8s.io/apimachinery/pkg/util/wait.loopConditionUntilContext.func1\n\tpkg/mod/k8s.io/[email protected]/pkg/util/wait/loop.go:53\nk8s.io/apimachinery/pkg/util/wait.loopConditionUntilContext\n\tpkg/mod/k8s.io/[email protected]/pkg/util/wait/loop.go:54\nk8s.io/apimachinery/pkg/util/wait.PollUntilContextCancel\n\tpkg/mod/k8s.io/[email protected]/pkg/util/wait/poll.go:33\nsigs.k8s.io/controller-runtime/pkg/internal/source.(*Kind[...]).Start.func1\n\tpkg/mod/sigs.k8s.io/[email protected]/pkg/internal/source/kind.go:64"} {"level":"error","ts":"2025-04-28T03:48:39Z","logger":"controller-runtime.source.EventHandler","msg":"failed to get informer from cache","error":"Timeout: failed waiting for *v1alpha1.NginxProxy Informer to sync","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/source.(*Kind[...]).Start.func1.1\n\tpkg/mod/sigs.k8s.io/[email protected]/pkg/internal/source/kind.go:76\nk8s.io/apimachinery/pkg/util/wait.loopConditionUntilContext.func1\n\tpkg/mod/k8s.io/[email protected]/pkg/util/wait/loop.go:53\nk8s.io/apimachinery/pkg/util/wait.loopConditionUntilContext\n\tpkg/mod/k8s.io/[email protected]/pkg/util/wait/loop.go:54\nk8s.io/apimachinery/pkg/util/wait.PollUntilContextCancel\n\tpkg/mod/k8s.io/[email protected]/pkg/util/wait/poll.go:33\nsigs.k8s.io/controller-runtime/pkg/internal/source.(*Kind[...]).Start.func1\n\tpkg/mod/sigs.k8s.io/[email protected]/pkg/internal/source/kind.go:64"} {"level":"error","ts":"2025-04-28T03:48:39Z","logger":"controller-runtime.source.EventHandler","msg":"failed to get informer from cache","error":"Timeout: failed waiting for *v1beta1.ReferenceGrant Informer to sync","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/source.(*Kind[...]).Start.func1.1\n\tpkg/mod/sigs.k8s.io/[email protected]/pkg/internal/source/kind.go:76\nk8s.io/apimachinery/pkg/util/wait.loopConditionUntilContext.func1\n\tpkg/mod/k8s.io/[email protected]/pkg/util/wait/loop.go:53\nk8s.io/apimachinery/pkg/util/wait.loopConditionUntilContext\n\tpkg/mod/k8s.io/[email protected]/pkg/util/wait/loop.go:54\nk8s.io/apimachinery/pkg/util/wait.PollUntilContextCancel\n\tpkg/mod/k8s.io/[email protected]/pkg/util/wait/poll.go:33\nsigs.k8s.io/controller-runtime/pkg/internal/source.(*Kind[...]).Start.func1\n\tpkg/mod/sigs.k8s.io/[email protected]/pkg/internal/source/kind.go:64"} {"level":"error","ts":"2025-04-28T03:48:39Z","logger":"controller-runtime.source.EventHandler","msg":"failed to get informer from cache","error":"Timeout: failed waiting for *v1.Gateway Informer to sync","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/source.(*Kind[...]).Start.func1.1\n\tpkg/mod/sigs.k8s.io/[email protected]/pkg/internal/source/kind.go:76\nk8s.io/apimachinery/pkg/util/wait.loopConditionUntilContext.func1\n\tpkg/mod/k8s.io/[email protected]/pkg/util/wait/loop.go:53\nk8s.io/apimachinery/pkg/util/wait.loopConditionUntilContext\n\tpkg/mod/k8s.io/[email protected]/pkg/util/wait/loop.go:54\nk8s.io/apimachinery/pkg/util/wait.PollUntilContextCancel\n\tpkg/mod/k8s.io/[email protected]/pkg/util/wait/poll.go:33\nsigs.k8s.io/controller-runtime/pkg/internal/source.(*Kind[...]).Start.func1\n\tpkg/mod/sigs.k8s.io/[email protected]/pkg/internal/source/kind.go:64"} {"level":"error","ts":"2025-04-28T03:48:39Z","logger":"controller-runtime.source.EventHandler","msg":"failed to get informer from cache","error":"Timeout: failed waiting for *v1alpha1.UpstreamSettingsPolicy Informer to sync","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/source.(*Kind[...]).Start.func1.1\n\tpkg/mod/sigs.k8s.io/[email protected]/pkg/internal/source/kind.go:76\nk8s.io/apimachinery/pkg/util/wait.loopConditionUntilContext.func1\n\tpkg/mod/k8s.io/[email protected]/pkg/util/wait/loop.go:53\nk8s.io/apimachinery/pkg/util/wait.loopConditionUntilContext\n\tpkg/mod/k8s.io/[email protected]/pkg/util/wait/loop.go:54\nk8s.io/apimachinery/pkg/util/wait.PollUntilContextCancel\n\tpkg/mod/k8s.io/[email protected]/pkg/util/wait/poll.go:33\nsigs.k8s.io/controller-runtime/pkg/internal/source.(*Kind[...]).Start.func1\n\tpkg/mod/sigs.k8s.io/[email protected]/pkg/internal/source/kind.go:64"} {"level":"info","ts":"2025-04-28T03:48:39Z","msg":"shutting down server","name":"health probe","addr":"[::]:8081"} {"level":"info","ts":"2025-04-28T03:48:39Z","logger":"controller-runtime.metrics","msg":"Shutting down metrics server with timeout of 1 minute"} failed to start control loop: failed to prepare the first batch: no matches for kind "ObservabilityPolicy" in version "gateway.nginx.org/v1alpha2"

To Reproduce
Steps to reproduce the behavior:
$kubectl kustomize "https://github.com/nginx/nginx-gateway-fabric/config/crd/gateway-api/standard?ref=v1.6.2" | kubectl apply -f -
$helm repo update $helm install ngf oci://ghcr.io/nginx/charts/nginx-gateway-fabric --create-namespace -n nginx-gateway --set service.type=NodePort
$kubectl wait --timeout=5m -n nginx-gateway deployment/ngf-nginx-gateway-fabric --for=condition=Available
#wait too long ctrl + c
$kubectl logs -n nginx-gateway ngf-nginx-gateway-fabric-5c4bfb8c48-s574v

Expected behavior
v1.6.2 shall run ok

Your environment

  • Version of the NGINX Gateway Fabric - release version or a specific commit. The first line of the nginx-gateway container logs includes the commit info.
    $ helm install ngf oci://ghcr.io/nginx/charts/nginx-gateway-fabric --create-namespace -n nginx-gateway --set service.type=NodePort Pulled: ghcr.io/nginx/charts/nginx-gateway-fabric:1.6.2 Digest: sha256:3a06aaa0a247b71becb37b4e44f030d17468d83bd537c282a3e368e2137e1765 NAME: ngf LAST DEPLOYED: Mon Apr 28 03:40:32 2025 NAMESPACE: nginx-gateway STATUS: deployed REVISION: 1 TEST SUITE: None

  • Version of Kubernetes
    $ kubectl version
    Client Version: v1.32.3
    Kustomize Version: v5.5.0
    Server Version: v1.32.3

  • Kubernetes platform (e.g. Mini-kube or GCP)
    self deployed cluster

  • Details on how you expose the NGINX Gateway Fabric Pod (e.g. Service of type LoadBalancer or port-forward)
    --set service.type=NodePort

  • Logs of NGINX container: kubectl -n nginx-gateway logs -l app=nginx-gateway -c nginx
    $ kubectl -n nginx-gateway logs -l app=nginx-gateway -c nginx
    No resources found in nginx-gateway namespace.

  • NGINX Configuration: kubectl -n nginx-gateway exec <gateway-pod> -c nginx -- nginx -T
    $ kubectl -n nginx-gateway exec ngf-nginx-gateway-fabric-5c4bfb8c48-s574v -c nginx -- nginx -T
    `2025/04/28 03:56:51 [notice] 83#83: js vm init njs: 00007FD3EDD37B00

configuration file /etc/nginx/nginx.conf:

load_module /usr/lib/nginx/modules/ngx_http_js_module.so;
include /etc/nginx/main-includes/*.conf;

worker_processes auto;

pid /var/run/nginx/nginx.pid;

events {
worker_connections 1024;
}

http {
include /etc/nginx/conf.d/*.conf;
include /etc/nginx/mime.types;
js_import /usr/lib/nginx/modules/njs/httpmatches.js;

default_type application/octet-stream;

proxy_headers_hash_bucket_size 512;
proxy_headers_hash_max_size 1024;
server_names_hash_bucket_size 256;
server_names_hash_max_size 1024;
variables_hash_bucket_size 512;
variables_hash_max_size 1024;

sendfile on;
tcp_nopush on;

server_tokens off;

server {
listen unix:/var/run/nginx/nginx-status.sock;
access_log off;

location /stub_status {
    stub_status;
}

}
}

stream {
variables_hash_bucket_size 512;
variables_hash_max_size 1024;

map_hash_max_size 2048;
map_hash_bucket_size 256;

log_format stream-main '$remote_addr [$time_local] '
'$protocol $status $bytes_sent $bytes_received '
'$session_time "$ssl_preread_server_name"';
access_log /dev/stdout stream-main;
include /etc/nginx/stream-conf.d/*.conf;
}

configuration file /etc/nginx/main-includes/main.conf:

error_log stderr info;

configuration file /etc/nginx/mime.types:

types {
text/html html htm shtml;
text/css css;
text/xml xml;
image/gif gif;
image/jpeg jpeg jpg;
application/javascript js;
application/atom+xml atom;
application/rss+xml rss;

text/mathml                                      mml;
text/plain                                       txt;
text/vnd.sun.j2me.app-descriptor                 jad;
text/vnd.wap.wml                                 wml;
text/x-component                                 htc;

image/avif                                       avif;
image/png                                        png;
image/svg+xml                                    svg svgz;
image/tiff                                       tif tiff;
image/vnd.wap.wbmp                               wbmp;
image/webp                                       webp;
image/x-icon                                     ico;
image/x-jng                                      jng;
image/x-ms-bmp                                   bmp;

font/woff                                        woff;
font/woff2                                       woff2;

application/java-archive                         jar war ear;
application/json                                 json;
application/mac-binhex40                         hqx;
application/msword                               doc;
application/pdf                                  pdf;
application/postscript                           ps eps ai;
application/rtf                                  rtf;
application/vnd.apple.mpegurl                    m3u8;
application/vnd.google-earth.kml+xml             kml;
application/vnd.google-earth.kmz                 kmz;
application/vnd.ms-excel                         xls;
application/vnd.ms-fontobject                    eot;
application/vnd.ms-powerpoint                    ppt;
application/vnd.oasis.opendocument.graphics      odg;
application/vnd.oasis.opendocument.presentation  odp;
application/vnd.oasis.opendocument.spreadsheet   ods;
application/vnd.oasis.opendocument.text          odt;
application/vnd.openxmlformats-officedocument.presentationml.presentation
                                                 pptx;
application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
                                                 xlsx;
application/vnd.openxmlformats-officedocument.wordprocessingml.document
                                                 docx;
application/vnd.wap.wmlc                         wmlc;
application/wasm                                 wasm;
application/x-7z-compressed                      7z;
application/x-cocoa                              cco;
application/x-java-archive-diff                  jardiff;
application/x-java-jnlp-file                     jnlp;
application/x-makeself                           run;
application/x-perl                               pl pm;
application/x-pilot                              prc pdb;
application/x-rar-compressed                     rar;
application/x-redhat-package-manager             rpm;
application/x-sea                                sea;
application/x-shockwave-flash                    swf;
application/x-stuffit                            sit;
application/x-tcl                                tcl tk;
application/x-x509-ca-cert                       der pem crt;
application/x-xpinstall                          xpi;
application/xhtml+xml                            xhtml;
application/xspf+xml                             xspf;
application/zip                                  zip;

application/octet-stream                         bin exe dnginx: the configuration file /etc/nginx/nginx.conf syntax is ok

nginx: configuration file /etc/nginx/nginx.conf test is successful
ll;
application/octet-stream deb;
application/octet-stream dmg;
application/octet-stream iso img;
application/octet-stream msi msp msm;

audio/midi                                       mid midi kar;
audio/mpeg                                       mp3;
audio/ogg                                        ogg;
audio/x-m4a                                      m4a;
audio/x-realaudio                                ra;

video/3gpp                                       3gpp 3gp;
video/mp2t                                       ts;
video/mp4                                        mp4;
video/mpeg                                       mpeg mpg;
video/quicktime                                  mov;
video/webm                                       webm;
video/x-flv                                      flv;
video/x-m4v                                      m4v;
video/x-mng                                      mng;
video/x-ms-asf                                   asx asf;
video/x-ms-wmv                                   wmv;
video/x-msvideo                                  avi;

}
`

Additional context
Debian OS
$ uname -a
Linux mathematicalsystems 6.12.21-amd64 #1 SMP PREEMPT_DYNAMIC Debian 6.12.21-1 (2025-03-30) x86_64 GNU/Linux
$ cat /etc/os-release
PRETTY_NAME="Debian GNU/Linux trixie/sid" NAME="Debian GNU/Linux" VERSION_CODENAME=trixie ID=debian HOME_URL="https://www.debian.org/" SUPPORT_URL="https://www.debian.org/support" BUG_REPORT_URL="https://bugs.debian.org/"

Metadata

Metadata

Assignees

No one assigned

    Labels

    Type

    No type

    Projects

    Status

    ✅ Done

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions