heroku migration with defang init (#1354)

* rough cut of PaaS import

* setup protos

* use fabric for llm invocations

* s/DeriveCompose/GenerateCompose/g

* retry if compose file is not valid yaml

* refactor to expose mockable surveyor interface

* upgrade secret-detector

* fix: correct retry behavior in generateComposeFile and fix test expectations

* remove secrets from config vars before sending to server

* update defang vendor hash

* remove version from generated compose file

* write compose file after generating it

* factor out surveyor package

* avoid proto getter

* handle windows newlines

* avoid stat before write--open with O_CREAT & O_EXCL

* use any instead of interface{}

* refactor extractFirstCodeBlock to return an error if code block is not found or incomplete

* use our http client, and reduce timeout

* refactor SourcePlatform type to support cobra help text

* prefix SourcePlatform

* rename proto enum avlue

* generate cmd ctx

* factor out handleGenerate

* combine new and setup commands

* use init as the primary name for new command

* factor out afterGenerate

* promote const

* remove redundant test if sample exists. cli.InitFromSamples already handles this

* promote default folder derivation

* extract promptForSample

* extract beforeGenerate

* partition sample and generate code paths

* invert handleGenerate so that cloneSample prompts and delegates to ai generate if necessary

* invoke cloneFromSample, aiGenerate, and migrateFromHerok in init

* fold beforeGenerateWithAI back into aiGenerate

* use a constant for tracking event name

* fold beforeGenerateFromSample back into cloneSample

* rename setup pkg to migrate

* move setup code into setup package

* fix panic

* add managed db tags to generate heroku compose file

* improve prompt for heroku auth token

Co-authored-by: Lio李歐 <[email protected]>

* preserve comments when cleaning yaml

* automatically generate heroku token

* interactive login before heroku migration

---------

Co-authored-by: Lio李歐 <[email protected]>

Author: jordanstephens

pkgs/defang/cli.nix

@@ -7,7 +7,7 @@ buildGoModule {
   pname = "defang-cli";
   version = "git";
   src = ../../src;
-  vendorHash = "sha256-4QMrneh4I2gTFf7erVnakqPDZakFzceGaN+ieAMNPX0="; # TODO: use fetchFromGitHub
+  vendorHash = "sha256-SfjkSc0Upaa12+GrRCpEqCSp4+6F/J+jEzVoE2ELdNY="; # TODO: use fetchFromGitHub
 
   subPackages = [ "cmd/cli" ];
 

src/cmd/cli/command/commands.go

@@ -9,7 +9,6 @@ import (
 	"os/exec"
 	"path/filepath"
 	"regexp"
-	"slices"
 	"strings"
 	"time"
 
@@ -23,7 +22,10 @@ import (
 	"github.com/DefangLabs/defang/src/pkg/clouds/aws"
 	"github.com/DefangLabs/defang/src/pkg/logs"
 	"github.com/DefangLabs/defang/src/pkg/mcp"
+	"github.com/DefangLabs/defang/src/pkg/migrate"
 	"github.com/DefangLabs/defang/src/pkg/scope"
+	"github.com/DefangLabs/defang/src/pkg/setup"
+	"github.com/DefangLabs/defang/src/pkg/surveyor"
 	"github.com/DefangLabs/defang/src/pkg/term"
 	"github.com/DefangLabs/defang/src/pkg/track"
 	"github.com/DefangLabs/defang/src/pkg/types"
@@ -43,6 +45,7 @@ var (
 	client         *cliClient.GrpcClient
 	cluster        string
 	colorMode      = ColorAuto
+	sourcePlatform = migrate.SourcePlatformUnspecified // default to auto-detecting the source platform
 	doDebug        = false
 	hasTty         = term.IsTerminal() && !pkg.GetenvBool("CI")
 	hideUpdate     = pkg.GetenvBool("DEFANG_HIDE_UPDATE")
@@ -227,7 +230,9 @@ func SetupCommands(ctx context.Context, version string) {
 	// Generate Command
 	generateCmd.Flags().StringVar(&modelId, "model", modelId, "LLM model to use for generating the code (Pro users only)")
 	RootCmd.AddCommand(generateCmd)
-	RootCmd.AddCommand(newCmd)
+	// new command
+	initCmd.PersistentFlags().Var(&sourcePlatform, "from", fmt.Sprintf(`the platform from which to migrate the project; one of %v`, migrate.AllSourcePlatforms))
+	RootCmd.AddCommand(initCmd)
 
 	// Get Services Command
 	lsCommand := makeComposePsCmd()
@@ -507,205 +512,101 @@ var certGenerateCmd = &cobra.Command{
 	},
 }
 
+func afterGenerate(ctx context.Context, result setup.SetupResult) {
+	term.Info("Code generated successfully in folder", result.Folder)
+	editor := pkg.Getenv("EDITOR", "code")
+	cmdd := exec.Command(editor, result.Folder)
+	err := cmdd.Start()
+	if err != nil {
+		term.Debugf("unable to launch editor %q: %v", editor, err)
+	}
+
+	cd := ""
+	if result.Folder != "." {
+		cd = "`cd " + result.Folder + "` and "
+	}
+
+	// Load the project and check for empty environment variables
+	loader := compose.NewLoader(compose.WithPath(filepath.Join(result.Folder, "compose.yaml")))
+	project, err := loader.LoadProject(ctx)
+	if err != nil {
+		term.Debugf("unable to load new project: %v", err)
+	}
+
+	var envInstructions []string
+	for _, envVar := range collectUnsetEnvVars(project) {
+		envInstructions = append(envInstructions, "config create "+envVar)
+	}
+
+	if len(envInstructions) > 0 {
+		printDefangHint("Check the files in your favorite editor.\nTo configure the service, do "+cd, envInstructions...)
+	} else {
+		printDefangHint("Check the files in your favorite editor.\nTo deploy the service, do "+cd, "compose up")
+	}
+}
+
 var generateCmd = &cobra.Command{
 	Use:     "generate",
 	Args:    cobra.MaximumNArgs(1),
 	Aliases: []string{"gen"},
 	Short:   "Generate a sample Defang project",
 	RunE: func(cmd *cobra.Command, args []string) error {
-		var sample, language, defaultFolder string
-		if len(args) > 0 {
-			sample = args[0]
-		}
+		ctx := cmd.Context()
 
-		if nonInteractive {
-			if sample == "" {
-				return errors.New("cannot run in non-interactive mode")
-			}
-			return cli.InitFromSamples(cmd.Context(), "", []string{sample})
-		}
-
-		sampleList, fetchSamplesErr := cli.FetchSamples(cmd.Context())
-		if sample == "" {
-			// Fetch the list of samples from the Defang repository
-			if fetchSamplesErr != nil {
-				term.Debug("unable to fetch samples:", fetchSamplesErr)
-			} else if len(sampleList) > 0 {
-				const generateWithAI = "Generate with AI"
-
-				sampleNames := []string{generateWithAI}
-				sampleTitles := []string{"Generate a sample from scratch using a language prompt"}
-				sampleIndex := []string{"unused first entry because we always show genAI option"}
-				for _, sample := range sampleList {
-					sampleNames = append(sampleNames, sample.Name)
-					sampleTitles = append(sampleTitles, sample.Title)
-					sampleIndex = append(sampleIndex, strings.ToLower(sample.Name+" "+sample.Title+" "+
-						strings.Join(sample.Tags, " ")+" "+strings.Join(sample.Languages, " ")))
-				}
-
-				if err := survey.AskOne(&survey.Select{
-					Message: "Choose a sample service:",
-					Options: sampleNames,
-					Help:    "The project code will be based on the sample you choose here.",
-					Filter: func(filter string, value string, i int) bool {
-						return i == 0 || strings.Contains(sampleIndex[i], strings.ToLower(filter))
-					},
-					Description: func(value string, i int) string {
-						return sampleTitles[i]
-					},
-				}, &sample, survey.WithStdio(term.DefaultTerm.Stdio())); err != nil {
-					return err
-				}
-				if sample == generateWithAI {
-					if err := survey.AskOne(&survey.Select{
-						Message: "Choose the language you'd like to use:",
-						Options: cli.SupportedLanguages,
-						Help:    "The project code will be in the language you choose here.",
-					}, &language, survey.WithStdio(term.DefaultTerm.Stdio())); err != nil {
-						return err
-					}
-					sample = ""
-					defaultFolder = "project1"
-				} else {
-					defaultFolder = sample
-				}
-			}
+		setupClient := setup.SetupClient{
+			Surveyor: surveyor.NewDefaultSurveyor(),
+			Heroku:   migrate.NewHerokuClient(),
+			ModelID:  modelId,
+			Fabric:   client,
+			Cluster:  getCluster(),
 		}
 
-		var qs = []*survey.Question{
-			{
-				Name: "description",
-				Prompt: &survey.Input{
-					Message: "Please describe the service you'd like to build:",
-					Help: `Here are some example prompts you can use:
-    "A simple 'hello world' function"
-    "A service with 2 endpoints, one to upload and the other to download a file from AWS S3"
-    "A service with a default endpoint that returns an HTML page with a form asking for the user's name and then a POST endpoint to handle the form post when the user clicks the 'submit' button"`,
-				},
-				Validate: survey.MinLength(5),
-			},
-			{
-				Name: "folder",
-				Prompt: &survey.Input{
-					Message: "What folder would you like to create the project in?",
-					Default: defaultFolder, // dynamically set based on chosen sample
-					Help:    "The generated code will be in the folder you choose here. If the folder does not exist, it will be created.",
-				},
-				Validate: survey.Required,
-			},
-		}
-
-		if sample != "" {
-			qs = qs[1:] // user picked a sample, so we skip the description question
-			sampleExists := slices.ContainsFunc(sampleList, func(s cli.Sample) bool {
-				return s.Name == sample
-			})
-
-			if !sampleExists {
-				return cli.ErrSampleNotFound
-			}
+		sample := ""
+		if len(args) > 0 {
+			sample = args[0]
 		}
-
-		prompt := struct {
-			Description string // or you can tag fields to match a specific name
-			Folder      string
-		}{}
-
-		// ask the remaining questions
-		err := survey.Ask(qs, &prompt, survey.WithStdio(term.DefaultTerm.Stdio()))
+		result, err := setupClient.CloneSample(ctx, sample)
 		if err != nil {
 			return err
 		}
+		afterGenerate(ctx, result)
+		return nil
+	},
+}
 
-		if client.CheckLoginAndToS(cmd.Context()) != nil {
-			// The user is either not logged in or has not agreed to the terms of service; ask for agreement to the terms now
-			if err := cli.InteractiveAgreeToS(cmd.Context(), client); err != nil {
-				// This might fail because the user did not log in. This is fine: server won't save the terms agreement, but can proceed with the generation
-				if connect.CodeOf(err) != connect.CodeUnauthenticated {
-					return err
-				}
-			}
-		}
-
-		track.Evt("Generate Started", P("language", language), P("sample", sample), P("description", prompt.Description), P("folder", prompt.Folder), P("model", modelId))
-
-		// Check if the current folder is empty
-		if empty, err := pkg.IsDirEmpty(prompt.Folder); !os.IsNotExist(err) && !empty {
-			nonEmptyFolder := fmt.Sprintf("The folder %q is not empty. We recommend running this command in an empty folder.", prompt.Folder)
-
-			var confirm bool
-			err := survey.AskOne(&survey.Confirm{
-				Message: nonEmptyFolder + " Continue creating project?",
-			}, &confirm, survey.WithStdio(term.DefaultTerm.Stdio()))
-			if err == nil && !confirm {
-				os.Exit(1)
-			}
-		}
-
-		if sample != "" {
-			term.Info("Fetching sample from the Defang repository...")
-			err := cli.InitFromSamples(cmd.Context(), prompt.Folder, []string{sample})
-			if err != nil {
-				return err
-			}
-		} else {
-			term.Info("Working on it. This may take 1 or 2 minutes...")
-			args := cli.GenerateArgs{
-				Description: prompt.Description,
-				Folder:      prompt.Folder,
-				Language:    language,
-				ModelId:     modelId,
-			}
-			_, err := cli.GenerateWithAI(cmd.Context(), client, args)
-			if err != nil {
-				return err
-			}
+var initCmd = &cobra.Command{
+	Use:     "init [SAMPLE]",
+	Args:    cobra.MaximumNArgs(1),
+	Aliases: []string{"new"},
+	Short:   "Create a new Defang project from a sample",
+	RunE: func(cmd *cobra.Command, args []string) error {
+		ctx := cmd.Context()
+		setupClient := setup.SetupClient{
+			Surveyor: surveyor.NewDefaultSurveyor(),
+			Heroku:   migrate.NewHerokuClient(),
+			ModelID:  modelId,
+			Fabric:   client,
+			Cluster:  getCluster(),
 		}
 
-		term.Info("Code generated successfully in folder", prompt.Folder)
-
-		editor := pkg.Getenv("DEFANG_EDITOR", "code") // TODO: should we use EDITOR env var instead?
-		cmdd := exec.Command(editor, prompt.Folder)
-		err = cmdd.Start()
-		if err != nil {
-			term.Debugf("unable to launch editor %q: %v", editor, err)
+		if len(args) > 0 {
+			_, err := setupClient.CloneSample(ctx, args[0])
+			return err
 		}
 
-		cd := ""
-		if prompt.Folder != "." {
-			cd = "`cd " + prompt.Folder + "` and "
+		if nonInteractive {
+			return errors.New("cannot run in non-interactive mode")
 		}
 
-		// Load the project and check for empty environment variables
-		loader := compose.NewLoader(compose.WithPath(filepath.Join(prompt.Folder, "compose.yaml")))
-		project, err := loader.LoadProject(cmd.Context())
+		result, err := setupClient.Start(ctx)
 		if err != nil {
-			term.Debugf("unable to load new project: %v", err)
-		}
-
-		var envInstructions []string
-		for _, envVar := range collectUnsetEnvVars(project) {
-			envInstructions = append(envInstructions, "config create "+envVar)
-		}
-
-		if len(envInstructions) > 0 {
-			printDefangHint("Check the files in your favorite editor.\nTo configure the service, do "+cd, envInstructions...)
-		} else {
-			printDefangHint("Check the files in your favorite editor.\nTo deploy the service, do "+cd, "compose up")
+			return err
 		}
-
+		afterGenerate(ctx, result)
 		return nil
 	},
 }
 
-var newCmd = &cobra.Command{
-	Use:     "new [SAMPLE]",
-	Args:    cobra.MaximumNArgs(1),
-	Aliases: []string{"init"},
-	Short:   "Create a new Defang project from a sample",
-	RunE:    generateCmd.RunE,
-}
-
 func collectUnsetEnvVars(project *composeTypes.Project) []string {
 	if project == nil {
 		return nil // in case loading failed

src/go.mod

@@ -16,7 +16,7 @@ require (
 	cloud.google.com/go/secretmanager v1.14.5
 	cloud.google.com/go/storage v1.50.0
 	github.com/AlecAivazis/survey/v2 v2.3.7
-	github.com/DefangLabs/secret-detector v0.0.0-20250108223530-c2b44d4c1f8f
+	github.com/DefangLabs/secret-detector v0.0.0-20250811234530-d4b4214cd679
 	github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883
 	github.com/aws/aws-sdk-go-v2 v1.32.6
 	github.com/aws/aws-sdk-go-v2/config v1.26.6
@@ -51,6 +51,8 @@ require (
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/cobra v1.8.0
 	github.com/spf13/pflag v1.0.6
+	github.com/stretchr/testify v1.10.0
+	go.yaml.in/yaml/v3 v3.0.4
 	golang.org/x/mod v0.18.0
 	golang.org/x/oauth2 v0.29.0
 	golang.org/x/sys v0.32.0
@@ -83,7 +85,6 @@ require (
 	github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
 	github.com/go-jose/go-jose/v4 v4.0.5 // indirect
 	github.com/go-viper/mapstructure/v2 v2.3.0 // indirect
-	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/s2a-go v0.1.9 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect
@@ -101,14 +102,14 @@ require (
 	github.com/santhosh-tekuri/jsonschema/v6 v6.0.1 // indirect
 	github.com/sergi/go-diff v1.3.1 // indirect
 	github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect
+	github.com/stretchr/objx v0.5.2 // indirect
 	github.com/xhit/go-str2duration/v2 v2.1.0 // indirect
 	github.com/yosida95/uritemplate/v3 v3.0.2 // indirect
 	github.com/zeebo/errs v1.4.0 // indirect
 	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
 	go.opentelemetry.io/contrib/detectors/gcp v1.34.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect
 	go.opentelemetry.io/otel/sdk/metric v1.35.0 // indirect
-	go.yaml.in/yaml/v3 v3.0.4 // indirect
 	golang.org/x/crypto v0.37.0 // indirect
 	golang.org/x/net v0.39.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20250422160041-2d3770c4ea7f // indirect

src/go.sum

@@ -36,8 +36,8 @@ github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOEl
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
 github.com/DefangLabs/cobra v1.8.0-defang h1:rTzAg1XbEk3yXUmQPumcwkLgi8iNCby5CjyG3sCwzKk=
 github.com/DefangLabs/cobra v1.8.0-defang/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0=
-github.com/DefangLabs/secret-detector v0.0.0-20250108223530-c2b44d4c1f8f h1:RTbUqLhPxejgK92ifVdMTIW9H23QLlscy8QXPDTfaL4=
-github.com/DefangLabs/secret-detector v0.0.0-20250108223530-c2b44d4c1f8f/go.mod h1:2UjtD/G/Sy2FxoHpxKnzHTXMpRURecwYal8HgbxcvkY=
+github.com/DefangLabs/secret-detector v0.0.0-20250811234530-d4b4214cd679 h1:qNT7R4qrN+5u5ajSbqSW1opHP4LA8lzA+ASyw5MQZjs=
+github.com/DefangLabs/secret-detector v0.0.0-20250811234530-d4b4214cd679/go.mod h1:blbwPQh4DTlCZEfk1BLU4oMIhLda2U+A840Uag9DsZw=
 github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.26.0 h1:f2Qw/Ehhimh5uO1fayV0QIW7DShEQqhtUfhYc+cBPlw=
 github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.26.0/go.mod h1:2bIszWvQRlJVmJLiuLhukLImRjKPcYdzzsx6darK02A=
 github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.50.0 h1:5IT7xOdq17MtcdtL/vtl6mGfzhaq4m4vpollPRmlsBQ=
@@ -164,8 +164,6 @@ github.com/go-viper/mapstructure/v2 v2.3.0 h1:27XbWsHIqhbdR5TIC911OfYvgSaW93HM+d
 github.com/go-viper/mapstructure/v2 v2.3.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
-github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
-github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
 github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8=
 github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
 github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
@@ -289,6 +287,8 @@ github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
 github.com/spiffe/go-spiffe/v2 v2.5.0 h1:N2I01KCUkv1FAjZXJMwh95KK1ZIQLYbPfhaxw8WS0hE=
 github.com/spiffe/go-spiffe/v2 v2.5.0/go.mod h1:P+NxobPc6wXhVtINNtFjNWGBTreew1GBUCwT2wPmb7g=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=