Add 503 status code when there are zero upstream endpoints (#3406)

Add 503 status code when there are zero upstream endpoints.

Problem: When using NGINX Plus, the generated state file for an upstream was not correctly generating the 503 server for upstreams with zero endpoints.

Solution: On upstreams with zero endpoints, generate the correct nginx conf in the state file.

Author: bjee19

internal/controller/nginx/agent/agent.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/nginx/nginx-gateway-fabric/internal/controller/nginx/agent/broadcast"
 	agentgrpc "github.com/nginx/nginx-gateway-fabric/internal/controller/nginx/agent/grpc"
+	"github.com/nginx/nginx-gateway-fabric/internal/controller/nginx/types"
 	"github.com/nginx/nginx-gateway-fabric/internal/controller/state/dataplane"
 	"github.com/nginx/nginx-gateway-fabric/internal/controller/state/resolver"
 	"github.com/nginx/nginx-gateway-fabric/internal/controller/status"
@@ -179,6 +180,16 @@ func buildStreamUpstreamServers(upstream dataplane.Upstream) *pb.UpdateStreamSer
 }
 
 func buildUpstreamServers(upstream dataplane.Upstream) []*structpb.Struct {
+	if len(upstream.Endpoints) == 0 {
+		return []*structpb.Struct{
+			{
+				Fields: map[string]*structpb.Value{
+					"server": structpb.NewStringValue(types.Nginx503Server),
+				},
+			},
+		}
+	}
+
 	servers := make([]*structpb.Struct, 0, len(upstream.Endpoints))
 
 	for _, endpoint := range upstream.Endpoints {

internal/controller/nginx/agent/agent_test.go

@@ -12,6 +12,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 
 	"github.com/nginx/nginx-gateway-fabric/internal/controller/nginx/agent/broadcast/broadcastfakes"
+	"github.com/nginx/nginx-gateway-fabric/internal/controller/nginx/types"
 	"github.com/nginx/nginx-gateway-fabric/internal/controller/state/dataplane"
 	"github.com/nginx/nginx-gateway-fabric/internal/controller/state/resolver"
 	"github.com/nginx/nginx-gateway-fabric/internal/controller/status"
@@ -179,6 +180,10 @@ func TestUpdateUpstreamServers(t *testing.T) {
 								},
 							},
 						},
+						{
+							Name:      "empty-upstream",
+							Endpoints: []resolver.Endpoint{},
+						},
 					},
 					StreamUpstreams: []dataplane.Upstream{
 						{
@@ -212,6 +217,20 @@ func TestUpdateUpstreamServers(t *testing.T) {
 							},
 						},
 					},
+					{
+						Action: &pb.NGINXPlusAction_UpdateHttpUpstreamServers{
+							UpdateHttpUpstreamServers: &pb.UpdateHTTPUpstreamServers{
+								HttpUpstreamName: "empty-upstream",
+								Servers: []*structpb.Struct{
+									{
+										Fields: map[string]*structpb.Value{
+											"server": structpb.NewStringValue(types.Nginx503Server),
+										},
+									},
+								},
+							},
+						},
+					},
 					{
 						Action: &pb.NGINXPlusAction_UpdateStreamServers{
 							UpdateStreamServers: &pb.UpdateStreamServers{
@@ -234,13 +253,14 @@ func TestUpdateUpstreamServers(t *testing.T) {
 				g.Expect(fakeBroadcaster.SendCallCount()).To(Equal(0))
 			} else if test.buildUpstreams {
 				g.Expect(deployment.GetNGINXPlusActions()).To(Equal(expActions))
-				g.Expect(fakeBroadcaster.SendCallCount()).To(Equal(2))
+				g.Expect(fakeBroadcaster.SendCallCount()).To(Equal(3))
 			}
 
 			if test.expErr {
 				expErr := errors.Join(
 					fmt.Errorf("couldn't update upstream via the API: %w", testErr),
 					fmt.Errorf("couldn't update upstream via the API: %w", testErr),
+					fmt.Errorf("couldn't update upstream via the API: %w", testErr),
 				)
 
 				g.Expect(deployment.GetLatestUpstreamError()).To(Equal(expErr))

internal/controller/nginx/config/upstreams.go

@@ -7,6 +7,7 @@ import (
 	"github.com/nginx/nginx-gateway-fabric/internal/controller/nginx/config/http"
 	"github.com/nginx/nginx-gateway-fabric/internal/controller/nginx/config/policies/upstreamsettings"
 	"github.com/nginx/nginx-gateway-fabric/internal/controller/nginx/config/stream"
+	"github.com/nginx/nginx-gateway-fabric/internal/controller/nginx/types"
 	"github.com/nginx/nginx-gateway-fabric/internal/controller/state/dataplane"
 	"github.com/nginx/nginx-gateway-fabric/internal/framework/helpers"
 )
@@ -17,8 +18,6 @@ var (
 )
 
 const (
-	// nginx503Server is used as a backend for services that cannot be resolved (have no IP address).
-	nginx503Server = "unix:/var/run/nginx/nginx-503-server.sock"
 	// nginx500Server is used as a server for the invalid backend ref upstream.
 	nginx500Server = "unix:/var/run/nginx/nginx-500-server.sock"
 	// invalidBackendRef is used as an upstream name for invalid backend references.
@@ -159,7 +158,7 @@ func (g GeneratorImpl) createUpstream(
 			StateFile: stateFile,
 			Servers: []http.UpstreamServer{
 				{
-					Address: nginx503Server,
+					Address: types.Nginx503Server,
 				},
 			},
 		}

internal/controller/nginx/config/upstreams_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/nginx/nginx-gateway-fabric/internal/controller/nginx/config/policies"
 	"github.com/nginx/nginx-gateway-fabric/internal/controller/nginx/config/policies/upstreamsettings"
 	"github.com/nginx/nginx-gateway-fabric/internal/controller/nginx/config/stream"
+	"github.com/nginx/nginx-gateway-fabric/internal/controller/nginx/types"
 	"github.com/nginx/nginx-gateway-fabric/internal/controller/state/dataplane"
 	"github.com/nginx/nginx-gateway-fabric/internal/controller/state/resolver"
 	"github.com/nginx/nginx-gateway-fabric/internal/framework/helpers"
@@ -216,7 +217,7 @@ func TestCreateUpstreams(t *testing.T) {
 			ZoneSize: ossZoneSize,
 			Servers: []http.UpstreamServer{
 				{
-					Address: nginx503Server,
+					Address: types.Nginx503Server,
 				},
 			},
 		},
@@ -277,7 +278,7 @@ func TestCreateUpstream(t *testing.T) {
 				ZoneSize: ossZoneSize,
 				Servers: []http.UpstreamServer{
 					{
-						Address: nginx503Server,
+						Address: types.Nginx503Server,
 					},
 				},
 			},
@@ -293,7 +294,7 @@ func TestCreateUpstream(t *testing.T) {
 				ZoneSize: ossZoneSize,
 				Servers: []http.UpstreamServer{
 					{
-						Address: nginx503Server,
+						Address: types.Nginx503Server,
 					},
 				},
 			},
@@ -581,7 +582,7 @@ func TestCreateUpstreamPlus(t *testing.T) {
 				StateFile: stateDir + "/no-endpoints.conf",
 				Servers: []http.UpstreamServer{
 					{
-						Address: nginx503Server,
+						Address: types.Nginx503Server,
 					},
 				},
 			},

internal/controller/nginx/types/types.go

@@ -0,0 +1,6 @@
+package types
+
+const (
+	// Nginx503Server is used as a backend for services that cannot be resolved (have no IP address).
+	Nginx503Server = "unix:/var/run/nginx/nginx-503-server.sock"
+)

tests/suite/graceful_recovery_test.go

@@ -585,9 +585,6 @@ func getUnexpectedNginxErrorLogs(nginxPodName, namespace string) string {
 		"connect() failed (111: Connection refused)",
 		"could not be resolved (host not found) during usage report",
 		"server returned 429",
-		// FIXME(salonichf5) remove this error message check
-		// when https://github.com/nginx/nginx-gateway-fabric/issues/2090 is completed.
-		"no live upstreams while connecting to upstream",
 	}
 
 	unexpectedErrors := ""