Merge branch 'main' into eric-add-postgres-support

Author: nullfunc

pkgs/defang/default.nix

@@ -10,22 +10,22 @@ system ? builtins.currentSystem
 }:
 let
   shaMap = {
-    x86_64-linux = "0zxd1hg10867j15j538hxx8iwh7ld3g9krsdw7ajz9h2b2dsa3pz";
-    aarch64-linux = "0334q80rg1945ymkq4r6a4bpmfiyxapgn82v9cgn9n9r5z688ymm";
-    x86_64-darwin = "00i3hyal33fgr0vgvwvxv4n1k0zdr2dml74qzd6ndhbnl4nv96cj";
-    aarch64-darwin = "00i3hyal33fgr0vgvwvxv4n1k0zdr2dml74qzd6ndhbnl4nv96cj";
+    x86_64-linux = "1z3n27w6vgc0xwncdnpi8lz44247k4k0c20a52mdwcwd7aj70xhj";
+    aarch64-linux = "0lfv1s9hx521dc1469p5myb4rffaifk6rnmhsgln7fg96kgjjjim";
+    x86_64-darwin = "18jfx4561p2ghrn74wp53xcbbc3xi3zxpdbl2f4npyip936snng7";
+    aarch64-darwin = "18jfx4561p2ghrn74wp53xcbbc3xi3zxpdbl2f4npyip936snng7";
   };
 
   urlMap = {
-    x86_64-linux = "https://github.com/DefangLabs/defang/releases/download/v0.5.32/defang_0.5.32_linux_amd64.tar.gz";
-    aarch64-linux = "https://github.com/DefangLabs/defang/releases/download/v0.5.32/defang_0.5.32_linux_arm64.tar.gz";
-    x86_64-darwin = "https://github.com/DefangLabs/defang/releases/download/v0.5.32/defang_0.5.32_macOS.zip";
-    aarch64-darwin = "https://github.com/DefangLabs/defang/releases/download/v0.5.32/defang_0.5.32_macOS.zip";
+    x86_64-linux = "https://github.com/DefangLabs/defang/releases/download/v0.5.36/defang_0.5.36_linux_amd64.tar.gz";
+    aarch64-linux = "https://github.com/DefangLabs/defang/releases/download/v0.5.36/defang_0.5.36_linux_arm64.tar.gz";
+    x86_64-darwin = "https://github.com/DefangLabs/defang/releases/download/v0.5.36/defang_0.5.36_macOS.zip";
+    aarch64-darwin = "https://github.com/DefangLabs/defang/releases/download/v0.5.36/defang_0.5.36_macOS.zip";
   };
 in
 stdenvNoCC.mkDerivation {
   pname = "defang";
-  version = "0.5.32";
+  version = "0.5.36";
   src = fetchurl {
     url = urlMap.${system};
     sha256 = shaMap.${system};

src/cmd/cli/command/commands.go

@@ -79,10 +79,14 @@ func Execute(ctx context.Context) error {
 			printDefangHint("Fix the error and try again. To validate the compose file, use:", compose+" config")
 		}
 
-		if strings.Contains(err.Error(), "secret") {
+		if strings.Contains(err.Error(), "config") {
 			printDefangHint("To manage sensitive service config, use:", "config")
 		}
 
+		if err.Error() == "resource_exhausted: maximum number of projects reached" {
+			printDefangHint("To deactivate a project, do:", "compose down")
+		}
+
 		var cerr *cli.CancelError
 		if errors.As(err, &cerr) {
 			printDefangHint("Detached. The process will keep running.\nTo continue the logs from where you left off, do:", cerr.Error())
@@ -820,7 +824,6 @@ var composeUpCmd = &cobra.Command{
 		var force, _ = cmd.Flags().GetBool("force")
 		var detach, _ = cmd.Flags().GetBool("detach")
 
-		targetState := defangv1.ServiceState_SERVICE_COMPLETED
 		since := time.Now()
 		deploy, project, err := cli.ComposeUp(cmd.Context(), client, force)
 		if err != nil {
@@ -839,35 +842,38 @@ var composeUpCmd = &cobra.Command{
 		}
 
 		tailCtx, cancelTail := context.WithCancelCause(cmd.Context())
-		defer cancelTail(nil) // to cancel waitServiceState
+		defer cancelTail(nil) // to cancel WaitServiceState and clean-up context
 
-		errSuccess := errors.New("deployment succeeded")
+		errCompleted := errors.New("deployment succeeded") // tail canceled because of deployment completion
+		const targetState = defangv1.ServiceState_DEPLOYMENT_COMPLETED
 		targetStateReached := false
 
 		go func() {
 			services := make([]string, len(deploy.Services))
 			for i, serviceInfo := range deploy.Services {
 				services[i] = serviceInfo.Service.Name
 			}
-			if err := waitServiceState(tailCtx, targetState, deploy.Etag, services); err != nil {
-				if errors.Is(err, errDeploymentFailed) {
+
+			if err := cli.WaitServiceState(tailCtx, client, targetState, deploy.Etag, services); err != nil {
+				var errDeploymentFailed cli.ErrDeploymentFailed
+				if errors.As(err, &errDeploymentFailed) {
 					cancelTail(err)
 				} else if !errors.Is(err, context.Canceled) {
-					term.Warnf("failed to wait for service status: %v", err)
+					term.Warnf("failed to wait for service status: %v", err) // TODO: don't print in Go-routine
 				}
 			} else {
 				targetStateReached = true
-				cancelTail(errSuccess)
+				cancelTail(errCompleted)
 			}
 		}()
 
 		// show users the current streaming logs
-		services := "all services"
+		tailSource := "all services"
 		if deploy.Etag != "" {
-			services = "deployment ID " + deploy.Etag
+			tailSource = "deployment ID " + deploy.Etag
 		}
 
-		term.Info("Tailing logs for", services, "; press Ctrl+C to detach:")
+		term.Info("Tailing logs for", tailSource, "; press Ctrl+C to detach:")
 		tailParams := cli.TailOptions{
 			Etag:  deploy.Etag,
 			Since: since,
@@ -876,29 +882,42 @@ var composeUpCmd = &cobra.Command{
 
 		// blocking call to tail
 		if err := cli.Tail(tailCtx, client, tailParams); err != nil {
-			if errors.Is(context.Cause(tailCtx), errDeploymentFailed) {
-				term.Warn("Deployment FAILED. Service(s) not running.")
+			term.Debugf("Tail failed with %v", err)
+			if connect.CodeOf(err) == connect.CodePermissionDenied {
+				// If tail fails because of missing permission, we wait for the deployment to finish
+				term.Warn("Unable to tail logs. Waiting for the deployment to finish.")
+				<-tailCtx.Done()
+			} else if !errors.Is(tailCtx.Err(), context.Canceled) {
+				return err // any error other than cancelation
+			}
 
-				_, isPlayground := client.(*cliClient.PlaygroundClient)
-				if !nonInteractive && isPlayground {
+			// Tail got canceled; if it was by anything other than completion, prompt to show debugger
+			if !errors.Is(context.Cause(tailCtx), errCompleted) {
+				var failedServices []string
+				var errDeploymentFailed cli.ErrDeploymentFailed
+				if errors.As(context.Cause(tailCtx), &errDeploymentFailed) {
+					term.Warn(errDeploymentFailed)
+					failedServices = []string{errDeploymentFailed.Service, errDeploymentFailed.Service + "-image"} // HACK: also grab Kaniko logs
+				} else {
+					term.Warn("Deployment is not finished. Service(s) might not be running.")
+					// TODO: some services might be OK and we should only debug the ones that are not
+				}
+
+				if _, isPlayground := client.(*cliClient.PlaygroundClient); !nonInteractive && isPlayground {
 					var aiDebug bool
 					if err := survey.AskOne(&survey.Confirm{
 						Message: "Would you like to debug the deployment with AI?",
 						Help:    "This will send logs and artifacts to our backend and attempt to diagnose the issue and provide a solution.",
 					}, &aiDebug); err != nil {
 						term.Debugf("failed to ask for AI debug: %v", err)
 					} else if aiDebug {
-						// Call the AI debug endpoint using the original command context (not the tailCtx which is canceled)
-						if err := cli.Debug(cmd.Context(), client, deploy.Etag, project.WorkingDir); err != nil {
+						// Call the AI debug endpoint using the original command context (not the tailCtx which is canceled); HACK: cmd might be canceled too
+						// TODO: use the WorkingDir of the failed service, might not be the project's root
+						if err := cli.Debug(context.TODO(), client, deploy.Etag, project.WorkingDir, failedServices); err != nil {
 							term.Warnf("failed to debug deployment: %v", err)
 						}
 					}
 				}
-			} else if connect.CodeOf(err) == connect.CodePermissionDenied {
-				// If tail fails because of missing permission, we wait for the deployment to finish
-				term.Warn("Failed to tail logs. Waiting for the deployment to finish.")
-				<-tailCtx.Done()
-			} else if !errors.Is(context.Cause(tailCtx), errSuccess) {
 				return err
 			}
 		}
@@ -952,7 +971,8 @@ var debugCmd = &cobra.Command{
 	RunE: func(cmd *cobra.Command, args []string) error {
 		etag, _ := cmd.Flags().GetString("etag")
 
-		return cli.Debug(cmd.Context(), client, etag, ".")
+		// TODO: use the WorkingDir of the current project instead of current folder
+		return cli.Debug(cmd.Context(), client, etag, ".", nil)
 	},
 }
 

src/cmd/cli/command/servicemonitor.go

@@ -262,6 +262,14 @@ func ValidateProject(project *compose.Project) error {
 			}
 		}
 
+		if _, ok := svccfg.Extensions["x-defang-postgres"]; ok {
+			// Ensure the image is a valid Postgres image
+			repo := strings.SplitN(svccfg.Image, ":", 2)[0]
+			if !strings.HasSuffix(repo, "postgres") {
+				term.Warnf("service %q: managed Postgres service should use a postgres image", svccfg.Name)
+			}
+		}
+
 		for k := range svccfg.Extensions {
 			switch k {
 			case "x-defang-dns-role", "x-defang-static-files", "x-defang-redis", "x-defang-postgres":

src/pkg/cli/compose/validation.go

@@ -2,6 +2,7 @@ package cli
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -11,39 +12,29 @@ import (
 	defangv1 "github.com/DefangLabs/defang/src/protos/io/defang/v1"
 )
 
-func Debug(ctx context.Context, c client.Client, etag, folder string) error {
+// Arbitrary limit on the maximum number of files to process to avoid walking the entire drive and we have limited
+// context window for the LLM also.
+// FIXME: Find a better way to handle files.
+const maxFiles = 20
+
+var ErrFileLimitReached = errors.New("file limit reached")
+
+func Debug(ctx context.Context, c client.Client, etag, folder string, services []string) error {
 	term.Debug("Invoking AI debugger for deployment", etag)
 
 	// FIXME: use the project information to determine which files to send
 	patterns := []string{"Dockerfile", "*compose.yaml", "*compose.yml", "*.js", "*.ts", "*.py", "*.go", "requirements.txt", "package.json", "go.mod"}
-	var files []*defangv1.File
-	for _, pattern := range patterns {
-		fullPattern := filepath.Join(folder, pattern)
-		matchingFiles, err := filepath.Glob(fullPattern)
-		if err != nil {
-			term.Debug("failed to find matching files:", err)
-			continue
-		}
-		for _, fullPath := range matchingFiles {
-			b, err := os.ReadFile(fullPath)
-			if err != nil {
-				term.Debug("failed to read file:", err)
-				continue
-			}
-			files = append(files, &defangv1.File{
-				Name:    filepath.Base(fullPath),
-				Content: string(b),
-			})
-		}
-	}
+
+	files := findMatchingFiles(folder, patterns)
 
 	if DoDryRun {
 		return ErrDryRun
 	}
 
 	resp, err := c.Debug(ctx, &defangv1.DebugRequest{
-		Etag:  etag,
-		Files: files,
+		Etag:     etag,
+		Files:    files,
+		Services: services,
 	})
 	if err != nil {
 		return err
@@ -67,7 +58,7 @@ func Debug(ctx context.Context, c client.Client, etag, folder string) error {
 
 		if (len(service.CodeChanges)) > 0 {
 			for _, changes := range service.CodeChanges {
-				term.Println(fmt.Sprintf("File %s:", changes.File))
+				term.Println(fmt.Sprintf("Updated %s:", changes.File))
 				term.Println("-------------------")
 				term.Println(changes.Change)
 				term.Println("")
@@ -80,3 +71,52 @@ func Debug(ctx context.Context, c client.Client, etag, folder string) error {
 	// }
 	return nil
 }
+
+func findMatchingFiles(folder string, patterns []string) []*defangv1.File {
+	var files []*defangv1.File
+	fileCount := 0
+
+	err := filepath.Walk(folder, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			term.Debug("error accessing path:", err)
+			return nil // continue walking
+		}
+
+		if info.IsDir() {
+			return nil // continue to next file/directory
+		}
+
+		if fileCount >= maxFiles {
+			term.Debug("file limit reached, stopping search")
+			return ErrFileLimitReached
+		}
+
+		for _, pattern := range patterns {
+			matched, err := filepath.Match(pattern, info.Name())
+			if err != nil {
+				term.Debug("error matching pattern:", err)
+				continue
+			}
+			if matched {
+				b, err := os.ReadFile(path)
+				if err != nil {
+					term.Debug("failed to read file:", err)
+					continue
+				}
+				files = append(files, &defangv1.File{
+					Name:    filepath.Base(path),
+					Content: string(b),
+				})
+				fileCount++
+				break // file matched, no need to check other patterns
+			}
+		}
+		return nil
+	})
+
+	if err != nil && err != ErrFileLimitReached {
+		term.Debug("error walking the path:", err)
+	}
+
+	return files
+}