fix: allow converting string literal block with list (#3050)

This PR updates the following: 
* Fix YAML String Block Conversion: Resolved an issue where multiline
strings (like tool descriptions) containing list syntax were being
re-encoded as double-quoted strings with explicit \n characters. They
now correctly use the | literal block style as expected.
```
description: |
  this is the description
  this tool uses the following parameter:
  - param_1
  - param_2

# will turn into 

description: "this is the description\nthis tool uses the following parameter:\n-param_1\n-param_2"
```
* Updated the converter to identify and retain initial comment
lines/license headers at the top of configuration files.
* Updated migration completion status to reflect "ended" state.
* Update to use "v1" -> "nested format" and "v2" -> "flat format"
* Remove "authSources" when checking for keys. We had previously removed
support for "authSources".

Fixes #3023

Author: Yuan325

cmd/internal/config.go

@@ -15,6 +15,7 @@
 package internal
 
 import (
+	"bufio"
 	"bytes"
 	"context"
 	"fmt"
@@ -120,14 +121,28 @@ func (p *ConfigParser) ParseConfig(ctx context.Context, raw []byte) (Config, err
 
 // ConvertConfig converts configuration file to flat format.
 func ConvertConfig(raw []byte) ([]byte, error) {
+	var buf bytes.Buffer
+	// Manually copy top-level comments and empty lines from the source
+	scanner := bufio.NewScanner(bytes.NewReader(raw))
+	for scanner.Scan() {
+		line := scanner.Text()
+		trimmed := strings.TrimSpace(line)
+
+		// If the line is a comment or whitespace, preserve it
+		if strings.HasPrefix(trimmed, "#") || trimmed == "" {
+			buf.WriteString(line + "\n")
+		} else {
+			// Stop at the first line of actual data
+			break
+		}
+	}
+
+	// convert configuration file to flat format
 	var input yaml.MapSlice
 	decoder := yaml.NewDecoder(bytes.NewReader(raw), yaml.UseOrderedMap())
+	encoder := yaml.NewEncoder(&buf, yaml.UseLiteralStyleIfMultiline(true))
 
-	// convert to config file v2
-	var buf bytes.Buffer
-	encoder := yaml.NewEncoder(&buf)
-
-	v1keys := []string{"sources", "authServices", "embeddingModels", "tools", "toolsets", "prompts"}
+	nestedFormatKey := []string{"sources", "authServices", "embeddingModels", "tools", "toolsets", "prompts"}
 	docIndex := 0
 	for {
 		if err := decoder.Decode(&input); err != nil {
@@ -142,59 +157,48 @@ func ConvertConfig(raw []byte) ([]byte, error) {
 			if !ok {
 				return nil, fmt.Errorf("doc %d: unexpected non-string key in input: %v", docIndex, item.Key)
 			}
-			// check if the key is config file v1's key
-			if slices.Contains(v1keys, key) {
-				// check if value conversion to yaml.MapSlice successfully
-				// fields such as "tools" in toolsets might pass the first check but
-				// fail to convert to MapSlice
-				if slice, ok := item.Value.(yaml.MapSlice); ok {
-					// Deprecated: convert authSources to authServices
-					switch key {
-					case "authSources", "authServices":
-						key = "authService"
-					case "sources":
-						key = "source"
-					case "embeddingModels":
-						key = "embeddingModel"
-					case "tools":
-						key = "tool"
-					case "toolsets":
-						key = "toolset"
-					case "prompts":
-						key = "prompt"
-					}
-					transformed, err := transformDocs(key, slice)
-					if err != nil {
-						return nil, fmt.Errorf("doc %d: invalid config format at key %q: %w", docIndex, key, err)
-					}
-					// encode per-doc
-					for _, doc := range transformed {
-						if err := encoder.Encode(doc); err != nil {
-							return nil, err
-						}
-					}
-				} else {
-					if hasKindField(input) {
-						// this doc is already v2, encode to buf
-						if err := encoder.Encode(input); err != nil {
-							return nil, err
-						}
-						break
-					}
-					return nil, fmt.Errorf("doc %d: invalid config format at key %q: expected map", docIndex, key)
-				}
-			} else {
-				// this doc is already v2, encode to buf
+			if hasKindField(input) {
+				// this doc is already in flat format, encode to buf
 				if err := encoder.Encode(input); err != nil {
 					return nil, err
 				}
 				break
 			}
+			// check if value conversion to yaml.MapSlice successfully
+			if slice, ok := item.Value.(yaml.MapSlice); slices.Contains(nestedFormatKey, key) && ok {
+				switch key {
+				case "authServices":
+					key = "authService"
+				case "sources":
+					key = "source"
+				case "embeddingModels":
+					key = "embeddingModel"
+				case "tools":
+					key = "tool"
+				case "toolsets":
+					key = "toolset"
+				case "prompts":
+					key = "prompt"
+				}
+				transformed, err := transformDocs(key, slice)
+				if err != nil {
+					return nil, fmt.Errorf("doc %d: invalid config format at key %q: %w", docIndex, key, err)
+				}
+				// encode per-doc
+				for _, doc := range transformed {
+					if err := encoder.Encode(doc); err != nil {
+						return nil, err
+					}
+				}
+			} else {
+				return nil, fmt.Errorf("doc %d: invalid config format at key %q: expected nested format keys and type map", docIndex, key)
+			}
 		}
 	}
 	return buf.Bytes(), nil
 }
 
+// hasKindField is a helper function to check if an input is in flat format
 func hasKindField(input yaml.MapSlice) bool {
 	for _, item := range input {
 		if key, ok := item.Key.(string); ok && key == "kind" {
@@ -204,7 +208,7 @@ func hasKindField(input yaml.MapSlice) bool {
 	return false
 }
 
-// transformDocs transforms the configuration file from v1 format to v2
+// transformDocs transforms the configuration file from nested to flat format
 // yaml.MapSlice will preserve the order in a map
 func transformDocs(kind string, input yaml.MapSlice) ([]yaml.MapSlice, error) {
 	var transformed []yaml.MapSlice

cmd/internal/config_test.go

@@ -184,7 +184,8 @@ func TestConvertConfig(t *testing.T) {
                     model: gemini-embedding-001
                     apiKey: some-key
                     dimension: 768`,
-			want: `kind: source
+			want: `
+kind: source
 name: my-pg-instance
 type: cloud-sql-postgres
 project: my-project
@@ -261,7 +262,8 @@ dimension: 768
             toolsets:
                 example_toolset:
                     - example_tool`,
-			want: `kind: tool
+			want: `
+kind: tool
 name: example_tool
 type: postgres-sql
 source: my-pg-instance
@@ -382,7 +384,8 @@ tools:
             kind: embeddingModel
             name: gemini-model2
             type: gemini`,
-			want: `kind: source
+			want: `
+kind: source
 name: my-pg-instance
 type: cloud-sql-postgres
 project: my-project
@@ -478,7 +481,8 @@ type: gemini
 		},
 		{
 			desc: "no convertion needed",
-			in: `kind: source
+			in: `
+kind: source
 name: my-pg-instance
 type: cloud-sql-postgres
 project: my-project
@@ -503,7 +507,8 @@ kind: toolset
 name: example_toolset
 tools:
 - example_tool`,
-			want: `kind: source
+			want: `
+kind: source
 name: my-pg-instance
 type: cloud-sql-postgres
 project: my-project
@@ -534,13 +539,13 @@ tools:
 			desc:   "invalid source",
 			in:     `sources: invalid`,
 			isErr:  true,
-			errStr: `doc 1: invalid config format at key "sources": expected map`,
+			errStr: `doc 1: invalid config format at key "sources": expected nested format keys and type map`,
 		},
 		{
 			desc:   "invalid toolset",
 			in:     `toolsets: invalid`,
 			isErr:  true,
-			errStr: `doc 1: invalid config format at key "toolsets": expected map`,
+			errStr: `doc 1: invalid config format at key "toolsets": expected nested format keys and type map`,
 		},
 	}
 	for _, tc := range tcs {

cmd/internal/migrate/command.go

@@ -65,7 +65,7 @@ func runMigrate(cmd *migrateCmd, opts *internal.ToolboxOptions) error {
 		return errMsg
 	}
 
-	logger.InfoContext(ctx, "migration process will start; any comments present in the original configuration files will not be preserved in the migrated files")
+	logger.InfoContext(ctx, "migration process will start; any comments (except for top-level comments) presented in the original configuration files will not be preserved in the migrated files")
 	var errs []error
 	// process each files independently.
 	for _, filePath := range filePaths {
@@ -127,7 +127,7 @@ func runMigrate(cmd *migrateCmd, opts *internal.ToolboxOptions) error {
 		}
 	}
 
-	logger.InfoContext(ctx, "migration completed!")
+	logger.InfoContext(ctx, "migration ended!")
 	// If errs is empty, errors.Join returns nil
 	return errors.Join(errs...)
 }

cmd/internal/migrate/command_test.go

@@ -52,7 +52,8 @@ sources:
     user: my_user
     password: my_pass`
 
-	toolsFileContentNew := `kind: source
+	toolsFileContentNew := `
+kind: source
 name: my-pg-instance
 type: cloud-sql-postgres
 project: my-project
@@ -74,7 +75,8 @@ tools:
         type: string
         description: some description`
 
-	toolsFileContent2New := `kind: tool
+	toolsFileContent2New := `
+kind: tool
 name: example_tool2
 type: postgres-sql
 source: my-pg-instance
@@ -268,7 +270,8 @@ sources:
     password: my_pass
 `
 
-	toolsFileContentNew := `kind: source
+	toolsFileContentNew := `
+kind: source
 name: my-pg-instance
 type: cloud-sql-postgres
 project: my-project
@@ -283,18 +286,23 @@ tools:
   example_tool2:
     kind: postgres-sql
     source: my-pg-instance
-    description: some description
+    description: |
+      some description
+      - string
     statement: SELECT * FROM SQL_STATEMENT;
     parameters:
       - name: country
         type: string
         description: some description`
 
-	toolsFileContent2New := `kind: tool
+	toolsFileContent2New := `
+kind: tool
 name: example_tool2
 type: postgres-sql
 source: my-pg-instance
-description: some description
+description: |
+  some description
+  - string
 statement: SELECT * FROM SQL_STATEMENT;
 parameters:
 - name: country