Support new Docker models (#1308)

lionello · web-flow · commit 62a6d5a3f978 · 2025-07-23T12:23:29.000-07:00
* Support new Docker models

* Update cli.nix vendorHash
diff --git a/pkgs/defang/cli.nix b/pkgs/defang/cli.nix
@@ -7,7 +7,7 @@ buildGoModule {
   pname = "defang-cli";
   version = "git";
   src = ../../src;
-  vendorHash = "sha256-VkI1L/yuZKRU8e4TeNZKXmB0hCxI+RvY8iqDlzWrY2s="; # TODO: use fetchFromGitHub
+  vendorHash = "sha256-4QMrneh4I2gTFf7erVnakqPDZakFzceGaN+ieAMNPX0="; # TODO: use fetchFromGitHub
 
   subPackages = [ "cmd/cli" ];
 
diff --git a/src/go.mod b/src/go.mod
@@ -32,7 +32,7 @@ require (
 	github.com/aws/smithy-go v1.22.1
 	github.com/awslabs/goformation/v7 v7.13.1
 	github.com/bufbuild/connect-go v1.10.0
-	github.com/compose-spec/compose-go/v2 v2.7.2-0.20250703132301-891fce532a51
+	github.com/compose-spec/compose-go/v2 v2.7.2-0.20250715094302-8da9902241f9
 	github.com/digitalocean/godo v1.131.1
 	github.com/docker/docker v25.0.6+incompatible
 	github.com/golang-jwt/jwt/v5 v5.2.2
@@ -108,6 +108,7 @@ require (
 	go.opentelemetry.io/contrib/detectors/gcp v1.34.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect
 	go.opentelemetry.io/otel/sdk/metric v1.35.0 // indirect
+	go.yaml.in/yaml/v3 v3.0.4 // indirect
 	golang.org/x/crypto v0.37.0 // indirect
 	golang.org/x/net v0.39.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20250422160041-2d3770c4ea7f // indirect
diff --git a/src/go.sum b/src/go.sum
@@ -114,8 +114,8 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42 h1:Om6kYQYDUk5wWbT0t0q6pvyM49i9XZAv9dDrkDA7gjk=
 github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8=
-github.com/compose-spec/compose-go/v2 v2.7.2-0.20250703132301-891fce532a51 h1:AjI75N9METifYMZK7eNt8XIgY9Sryv+1w3XDA7X2vZQ=
-github.com/compose-spec/compose-go/v2 v2.7.2-0.20250703132301-891fce532a51/go.mod h1:Zow/3eYNOnl2T4qLGZEizf8d/ht1qfy09G7WGOSzGOY=
+github.com/compose-spec/compose-go/v2 v2.7.2-0.20250715094302-8da9902241f9 h1:kqvhWCmg3fVAPbfE8aJdV+qX1VqK4oK/DRI5yxeVd4E=
+github.com/compose-spec/compose-go/v2 v2.7.2-0.20250715094302-8da9902241f9/go.mod h1:veko/VB7URrg/tKz3vmIAQDaz+CGiXH8vZsW79NmAww=
 github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
 github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
 github.com/cpuguy83/go-md2man/v2 v2.0.6 h1:XJtiaUW6dEEqVuZiMTn1ldk455QWwEIsMIJlo5vtkx0=
@@ -329,6 +329,8 @@ go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt
 go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc=
 go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I=
 go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM=
+go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
+go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
diff --git a/src/pkg/cli/compose/context.go b/src/pkg/cli/compose/context.go
@@ -180,11 +180,11 @@ func getRemoteBuildContext(ctx context.Context, provider client.Provider, projec
 	}
 
 	var archiveType ArchiveType
-	// If we have a Railpack build, we use a zip archive
 	if build.Dockerfile == RAILPACK {
+		// If we have a Railpack build, we use a zip archive
 		archiveType = ArchiveTypeZip
-		// We use tar for all other builds
 	} else {
+		// We use gzip tar for all other builds
 		archiveType = ArchiveTypeGzip
 	}
 
@@ -363,8 +363,8 @@ func WalkContextFolder(root, dockerfile string, fn func(path string, de os.DirEn
 
 		slashPath := filepath.ToSlash(relPath)
 
-		// we need the Dockerfile, even if it's in the .dockerignore file
 		if relPath == dockerfile {
+			// we need the Dockerfile, even if it's in the .dockerignore file
 		} else if relPath == dockerignore {
 			// we need the .dockerignore file too: it might ignore itself and/or the Dockerfile, but is needed by the builder
 		} else {
diff --git a/src/pkg/cli/compose/context_test.go b/src/pkg/cli/compose/context_test.go
@@ -75,7 +75,6 @@ func TestUploadArchive(t *testing.T) {
 				t.Errorf("Expected %v, got %v", server.URL+path, url)
 			}
 		})
-
 	})
 
 	t.Run("force upload without digest", func(t *testing.T) {
diff --git a/src/pkg/cli/compose/fixup.go b/src/pkg/cli/compose/fixup.go
@@ -12,13 +12,12 @@ import (
 	"github.com/DefangLabs/defang/src/pkg/cli/client"
 	"github.com/DefangLabs/defang/src/pkg/term"
 	defangv1 "github.com/DefangLabs/defang/src/protos/io/defang/v1"
-	"github.com/compose-spec/compose-go/v2/types"
 	composeTypes "github.com/compose-spec/compose-go/v2/types"
 )
 
 const RAILPACK = "*Railpack"
 
-func FixupServices(ctx context.Context, provider client.Provider, project *types.Project, upload UploadMode) error {
+func FixupServices(ctx context.Context, provider client.Provider, project *composeTypes.Project, upload UploadMode) error {
 	// Preload the current config so we can detect which environment variables should be passed as "secrets"
 	config, err := provider.ListConfig(ctx, &defangv1.ListConfigsRequest{Project: project.Name})
 	if err != nil {
@@ -69,6 +68,12 @@ func FixupServices(ctx context.Context, provider client.Provider, project *types
 		project.Services[svccfg.Name] = svccfg
 	}
 
+	for name, model := range project.Models {
+		model.Name = name // ensure the model has a name
+		svccfg := fixupModel(model, project)
+		project.Services[svccfg.Name] = *svccfg
+	}
+
 	svcNameReplacer := NewServiceNameReplacer(provider, project)
 
 	for _, svccfg := range project.Services {
@@ -189,17 +194,17 @@ func parsePortString(port string) (uint32, error) {
 	}
 }
 
-func fixupLLM(svccfg *types.ServiceConfig) {
+func fixupLLM(svccfg *composeTypes.ServiceConfig) {
 	image := getImageRepo(svccfg.Image)
 	if strings.HasSuffix(image, "/openai-access-gateway") && len(svccfg.Ports) == 0 {
 		// HACK: we must have at least one host port to get a CNAME for the service
 		var port uint32 = 80
 		term.Debugf("service %q: adding LLM host port %d", svccfg.Name, port)
-		svccfg.Ports = []types.ServicePortConfig{{Target: port, Mode: Mode_HOST, Protocol: Protocol_TCP}}
+		svccfg.Ports = []composeTypes.ServicePortConfig{{Target: port, Mode: Mode_HOST, Protocol: Protocol_TCP}}
 	}
 }
 
-func fixupPostgresService(svccfg *types.ServiceConfig, provider client.Provider, upload UploadMode) error {
+func fixupPostgresService(svccfg *composeTypes.ServiceConfig, provider client.Provider, upload UploadMode) error {
 	if _, ok := provider.(*client.PlaygroundProvider); ok && upload != UploadModeEstimate {
 		term.Warnf("service %q: managed postgres is not supported in the Playground; consider using BYOC (https://s.defang.io/byoc)", svccfg.Name)
 	}
@@ -215,12 +220,12 @@ func fixupPostgresService(svccfg *types.ServiceConfig, provider client.Provider,
 			}
 		}
 		term.Debugf("service %q: adding postgres host port %d", svccfg.Name, port)
-		svccfg.Ports = []types.ServicePortConfig{{Target: port, Mode: Mode_HOST, Protocol: Protocol_TCP}}
+		svccfg.Ports = []composeTypes.ServicePortConfig{{Target: port, Mode: Mode_HOST, Protocol: Protocol_TCP}}
 	}
 	return nil
 }
 
-func fixupMongoService(svccfg *types.ServiceConfig, provider client.Provider, upload UploadMode) error {
+func fixupMongoService(svccfg *composeTypes.ServiceConfig, provider client.Provider, upload UploadMode) error {
 	if _, ok := provider.(*client.PlaygroundProvider); ok && upload != UploadModeEstimate {
 		term.Warnf("service %q: managed mongodb is not supported in the Playground; consider using BYOC (https://s.defang.io/byoc)", svccfg.Name)
 	}
@@ -250,12 +255,12 @@ func fixupMongoService(svccfg *types.ServiceConfig, provider client.Provider, up
 			break // done
 		}
 		term.Debugf("service %q: adding mongodb host port %d", svccfg.Name, port)
-		svccfg.Ports = []types.ServicePortConfig{{Target: port, Mode: Mode_HOST, Protocol: Protocol_TCP}}
+		svccfg.Ports = []composeTypes.ServicePortConfig{{Target: port, Mode: Mode_HOST, Protocol: Protocol_TCP}}
 	}
 	return nil
 }
 
-func fixupRedisService(svccfg *types.ServiceConfig, provider client.Provider, upload UploadMode) error {
+func fixupRedisService(svccfg *composeTypes.ServiceConfig, provider client.Provider, upload UploadMode) error {
 	if _, ok := provider.(*client.PlaygroundProvider); ok && upload != UploadModeEstimate {
 		term.Warnf("service %q: Managed redis is not supported in the Playground; consider using BYOC (https://s.defang.io/byoc)", svccfg.Name)
 	}
@@ -275,55 +280,102 @@ func fixupRedisService(svccfg *types.ServiceConfig, provider client.Provider, up
 			}
 		}
 		term.Debugf("service %q: adding redis host port %d", svccfg.Name, port)
-		svccfg.Ports = []types.ServicePortConfig{{Target: port, Mode: Mode_HOST, Protocol: Protocol_TCP}}
+		svccfg.Ports = []composeTypes.ServicePortConfig{{Target: port, Mode: Mode_HOST, Protocol: Protocol_TCP}}
 	}
 	return nil
 }
 
-func fixupModelProvider(svccfg *types.ServiceConfig, project *types.Project) {
-	// Declare a private network for the model provider
-	const modelProviderNetwork = "model_provider_private"
+// Declare a private network for the model provider
+const modelProviderNetwork = "model_provider_private"
+
+func fixupModel(model composeTypes.ModelConfig, project *composeTypes.Project) *composeTypes.ServiceConfig {
+	svccfg := &composeTypes.ServiceConfig{
+		Name:       model.Name,
+		Extensions: model.Extensions,
+	}
+	makeAccessGatewayService(svccfg, project, model.Model) // TODO: pass other model options too
+	return svccfg
+}
+
+func fixupModelProvider(svccfg *composeTypes.ServiceConfig, project *composeTypes.Project) {
+	var model string
+	if modelVals := svccfg.Provider.Options["model"]; len(modelVals) == 1 {
+		model = modelVals[0]
+	}
+	makeAccessGatewayService(svccfg, project, model)
+}
 
+func makeAccessGatewayService(svccfg *composeTypes.ServiceConfig, project *composeTypes.Project, model string) {
 	// Local Docker sets [SERVICE]_URL and [SERVICE]_MODEL environment variables on the dependent services
 	envName := strings.ToUpper(svccfg.Name) // TODO: handle characters that are not allowed in env vars, like '-'
-	urlEnv := envName + "_URL"
+	endpointEnvVar := envName + "_URL"
 	urlVal := "http://" + svccfg.Name + "/api/v1/"
-	modelEnvKey := envName + "_MODEL"
-	modelVals := svccfg.Provider.Options["model"]
+	modelEnvVar := envName + "_MODEL"
 
 	empty := ""
-	// svccfg.Deploy.Resources.Reservations.Limits = &types.Resources{} TODO: avoid memory limits warning
+	// svccfg.Deploy.Resources.Reservations.Limits = &composeTypes.Resources{} TODO: avoid memory limits warning
 	if svccfg.Environment == nil {
-		svccfg.Environment = types.MappingWithEquals{}
+		svccfg.Environment = composeTypes.MappingWithEquals{}
 	}
 	if _, exists := svccfg.Environment["OPENAI_API_KEY"]; !exists {
 		svccfg.Environment["OPENAI_API_KEY"] = &empty // disable auth; see https://github.com/DefangLabs/openai-access-gateway/pull/5
 	}
-	// svccfg.HealthCheck = &types.ServiceHealthCheckConfig{} TODO: add healthcheck
+	// svccfg.HealthCheck = &composeTypes.ServiceHealthCheckConfig{} TODO: add healthcheck
 	svccfg.Image = "defangio/openai-access-gateway"
 	if svccfg.Networks == nil {
 		// New compose-go versions do not create networks for "provider:" services, so we need to create it here
-		svccfg.Networks = make(map[string]*types.ServiceNetworkConfig)
+		svccfg.Networks = make(map[string]*composeTypes.ServiceNetworkConfig)
 	} else {
 		delete(svccfg.Networks, "default") // remove the default network
 	}
 	svccfg.Networks[modelProviderNetwork] = nil
-	svccfg.Ports = []types.ServicePortConfig{{Target: 80, Mode: Mode_HOST, Protocol: Protocol_TCP}}
+	svccfg.Ports = []composeTypes.ServicePortConfig{{Target: 80, Mode: Mode_HOST, Protocol: Protocol_TCP}}
 	svccfg.Provider = nil // remove "provider:" because current backend will not accept it
-	project.Networks[modelProviderNetwork] = types.NetworkConfig{Name: modelProviderNetwork}
+	project.Networks[modelProviderNetwork] = composeTypes.NetworkConfig{Name: modelProviderNetwork}
 
-	// Set environment variables (url and model) for any service that depends on the provider pseudo service
+	// Set environment variables (url and model) for any service that depends on the model
 	for _, dependency := range project.Services {
 		if _, ok := dependency.DependsOn[svccfg.Name]; ok {
 			if dependency.Environment == nil {
-				dependency.Environment = make(types.MappingWithEquals)
+				dependency.Environment = make(composeTypes.MappingWithEquals)
+			}
+			dependency.Networks[modelProviderNetwork] = nil
+			if _, ok := dependency.Environment[endpointEnvVar]; !ok {
+				dependency.Environment[endpointEnvVar] = &urlVal
+			}
+			if _, ok := dependency.Environment[modelEnvVar]; !ok && model != "" {
+				dependency.Environment[modelEnvVar] = &model
+			}
+		}
+
+		if modelDep, ok := dependency.Models[svccfg.Name]; ok {
+			endpointVar := endpointEnvVar
+			if modelDep != nil && modelDep.EndpointVariable != "" {
+				endpointVar = modelDep.EndpointVariable
+			}
+			modelVar := modelEnvVar
+			if modelDep != nil && modelDep.ModelVariable != "" {
+				modelVar = modelDep.ModelVariable
+			}
+			if dependency.Environment == nil {
+				dependency.Environment = make(composeTypes.MappingWithEquals)
 			}
 			dependency.Networks[modelProviderNetwork] = nil
-			if _, ok := dependency.Environment[urlEnv]; !ok {
-				dependency.Environment[urlEnv] = &urlVal
+			if _, ok := dependency.Environment[endpointVar]; !ok {
+				dependency.Environment[endpointVar] = &urlVal
+			}
+			if _, ok := dependency.Environment[modelVar]; !ok && model != "" {
+				dependency.Environment[modelVar] = &model
 			}
-			if _, ok := dependency.Environment[modelEnvKey]; !ok && len(modelVals) == 1 {
-				dependency.Environment[modelEnvKey] = &modelVals[0]
+			// If the model is not already declared as a dependency, add it
+			if _, ok := dependency.DependsOn[svccfg.Name]; !ok {
+				if dependency.DependsOn == nil {
+					dependency.DependsOn = make(map[string]composeTypes.ServiceDependency)
+				}
+				dependency.DependsOn[svccfg.Name] = composeTypes.ServiceDependency{
+					Condition: composeTypes.ServiceConditionStarted,
+					Required:  true,
+				}
 			}
 		}
 	}
diff --git a/src/pkg/cli/compose/validation.go b/src/pkg/cli/compose/validation.go
@@ -53,7 +53,7 @@ func ValidateProject(project *composeTypes.Project) error {
 				continue
 			}
 			if gcp.SafeLabelValue(svccfg.Name) == gcp.SafeLabelValue(services[j].Name) { // TODO: Shouldn't be just gcp specific
-				errs = append(errs, fmt.Errorf("The service names %q and %q normalize to the same value, which causes a conflict. Please use distinct names that differ after normalization", svccfg.Name, services[j].Name))
+				errs = append(errs, fmt.Errorf("the service names %q and %q normalize to the same value, which causes a conflict. Please use distinct names that differ after normalization", svccfg.Name, services[j].Name))
 			}
 		}
 	}
diff --git a/src/testdata/models/compose.yaml b/src/testdata/models/compose.yaml
@@ -0,0 +1,20 @@
+services:
+  app:
+    image: app
+    models: # per docs, should support list as well
+      ai_model: {}
+  withendpoint:
+    image: app
+    models:
+      my_model:
+        endpoint_var: MODEL_URL
+
+models:
+  ai_model:
+    model: ai/model
+  my_model:
+    model: ai/model
+    context_size: 1024
+    runtime_flags:
+      - "--a-flag"
+      - "--another-flag=42"
diff --git a/src/testdata/models/compose.yaml.fixup b/src/testdata/models/compose.yaml.fixup
@@ -0,0 +1,72 @@
+{
+  "ai_model": {
+    "command": null,
+    "entrypoint": null,
+    "environment": {
+      "OPENAI_API_KEY": ""
+    },
+    "image": "defangio/openai-access-gateway",
+    "networks": {
+      "model_provider_private": null
+    },
+    "ports": [
+      {
+        "mode": "host",
+        "target": 80,
+        "protocol": "tcp"
+      }
+    ]
+  },
+  "app": {
+    "command": null,
+    "entrypoint": null,
+    "environment": {
+      "AI_MODEL_MODEL": "ai/model",
+      "AI_MODEL_URL": "http://ai-model/api/v1/"
+    },
+    "image": "app",
+    "models": {
+      "ai_model": {}
+    },
+    "networks": {
+      "default": null,
+      "model_provider_private": null
+    }
+  },
+  "my_model": {
+    "command": null,
+    "entrypoint": null,
+    "environment": {
+      "OPENAI_API_KEY": ""
+    },
+    "image": "defangio/openai-access-gateway",
+    "networks": {
+      "model_provider_private": null
+    },
+    "ports": [
+      {
+        "mode": "host",
+        "target": 80,
+        "protocol": "tcp"
+      }
+    ]
+  },
+  "withendpoint": {
+    "command": null,
+    "entrypoint": null,
+    "environment": {
+      "MODEL_URL": "http://my-model/api/v1/",
+      "MY_MODEL_MODEL": "ai/model"
+    },
+    "image": "app",
+    "models": {
+      "my_model": {
+        "endpoint_var": "MODEL_URL"
+      }
+    },
+    "networks": {
+      "default": null,
+      "model_provider_private": null
+    }
+  }
+}
diff --git a/src/testdata/models/compose.yaml.golden b/src/testdata/models/compose.yaml.golden
diff --git a/src/testdata/models/compose.yaml.warnings b/src/testdata/models/compose.yaml.warnings

Original file line number	Diff line number	Diff line change
`@@ -75,7 +75,6 @@ func TestUploadArchive(t *testing.T) {`
`75`	`75`	`t.Errorf("Expected %v, got %v", server.URL+path, url)`
`76`	`76`	`}`
`77`	`77`	`})`
`78`		`-`
`79`	`78`	`})`
`80`	`79`
`81`	`80`	`t.Run("force upload without digest", func(t *testing.T) {`
Original file line number	Diff line number	Diff line change
`@@ -53,7 +53,7 @@ func ValidateProject(project *composeTypes.Project) error {`
`53`	`53`	`continue`
`54`	`54`	`}`
`55`	`55`	`if gcp.SafeLabelValue(svccfg.Name) == gcp.SafeLabelValue(services[j].Name) { // TODO: Shouldn't be just gcp specific`
`56`		`- errs = append(errs, fmt.Errorf("The service names %q and %q normalize to the same value, which causes a conflict. Please use distinct names that differ after normalization", svccfg.Name, services[j].Name))`
	`56`	`+ errs = append(errs, fmt.Errorf("the service names %q and %q normalize to the same value, which causes a conflict. Please use distinct names that differ after normalization", svccfg.Name, services[j].Name))`
`57`	`57`	`}`
`58`	`58`	`}`
`59`	`59`	`}`