fix: Rust comment merge

--bug=1

Author: studyzy

adapters/rust/context.go

@@ -44,6 +44,11 @@ func FindOwnerNode(commentNode *sitter.Node, src []byte) *sitter.Node {
 				curr = curr.NextNamedSibling()
 				continue
 			}
+			// Skip other comments to find the actual item they are documenting
+			if curr.Type() == "line_comment" || curr.Type() == "block_comment" {
+				curr = curr.NextNamedSibling()
+				continue
+			}
 			return curr
 		}
 		// Fallback to parent if no sibling found (orphaned doc comment)

adapters/rust/extractor.go

@@ -32,10 +32,6 @@ func (a *RustAdapter) extractComments(root *sitter.Node, src []byte, file string
 			node := c.Node
 			content := node.Content(src)
 
-			if isEmptyComment(content) {
-				continue
-			}
-
 			// Find Owner and Symbol Path
 			owner := FindOwnerNode(node, src)
 			symbolPath := ResolveSymbolPath(owner, src)
@@ -79,7 +75,17 @@ func (a *RustAdapter) extractComments(root *sitter.Node, src []byte, file string
 		}
 	}
 
-	return mergeComments(comments), nil
+	merged := mergeComments(comments)
+
+	// Filter empty comments after merge
+	var finalComments []*domain.Comment
+	for _, c := range merged {
+		if !isEmptyComment(c.SourceText) {
+			finalComments = append(finalComments, c)
+		}
+	}
+
+	return finalComments, nil
 }
 
 // mergeComments merges consecutive comments of the same type and symbol.
@@ -143,20 +149,39 @@ func getDomainCommentType(content string) domain.CommentType {
 
 func isEmptyComment(content string) bool {
 	trimmed := strings.TrimSpace(content)
-	switch {
-	case strings.HasPrefix(trimmed, "///"):
-		return strings.TrimSpace(strings.TrimPrefix(trimmed, "///")) == ""
-	case strings.HasPrefix(trimmed, "//!"):
-		return strings.TrimSpace(strings.TrimPrefix(trimmed, "//!")) == ""
-	case strings.HasPrefix(trimmed, "//"):
-		return strings.TrimSpace(strings.TrimPrefix(trimmed, "//")) == ""
-	case strings.HasPrefix(trimmed, "/*"):
+	
+	// Block comments
+	if strings.HasPrefix(trimmed, "/*") {
 		inner := strings.TrimPrefix(trimmed, "/*")
 		if strings.HasSuffix(inner, "*/") {
 			inner = strings.TrimSuffix(inner, "*/")
 		}
 		return strings.TrimSpace(inner) == ""
-	default:
-		return strings.TrimSpace(trimmed) == ""
 	}
+
+	// Line and Doc comments (check each line)
+	lines := strings.Split(content, "\n")
+	for _, line := range lines {
+		t := strings.TrimSpace(line)
+		if t == "" {
+			continue
+		}
+
+		if strings.HasPrefix(t, "///") {
+			t = strings.TrimPrefix(t, "///")
+		} else if strings.HasPrefix(t, "//!") {
+			t = strings.TrimPrefix(t, "//!")
+		} else if strings.HasPrefix(t, "//") {
+			t = strings.TrimPrefix(t, "//")
+		} else {
+			// Not a standard line comment marker, assume content
+			return false
+		}
+
+		if strings.TrimSpace(t) != "" {
+			return false
+		}
+	}
+	
+	return true
 }

adapters/translator/llm.go

@@ -42,7 +42,8 @@ func (t *LLMTranslator) Translate(ctx context.Context, text, from, to string) (s
 			"1. Keep technical terms, variable names, and code snippets unchanged.\n"+
 			"2. Maintain the tone and style of the original comment.\n"+
 			"3. Output ONLY the translated text, no explanations or quotes.\n"+
-			"4. If the text is already in the target language, return it as is.\n\n"+
+			"4. If the text is already in the target language, return it as is.\n"+
+			"5. Preserve all line breaks and formatting.\n\n"+
 			"Original: %s",
 		from, to, text,
 	)
@@ -139,7 +140,8 @@ func (t *LLMTranslator) buildBatchPrompt(texts []string, from, to string) string
 			"2. The output must be a valid JSON string array [\"...\",\"...\"].\n"+
 			"3. The number of elements MUST match the input.\n"+
 			"4. Keep technical terms, variable names, and code snippets unchanged.\n"+
-			"5. If a comment is already in the target language, return it as is.\n\n"+
+			"5. If a comment is already in the target language, return it as is.\n"+
+			"6. Preserve all line breaks and formatting.\n\n"+
 			"Input:\n%s",
 		from, to, string(inputJSON),
 	)

adapters/translator/newline_test.go

@@ -0,0 +1,54 @@
+package translator
+
+import (
+	"context"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestTranslateNewlinePreservation(t *testing.T) {
+	apiKey := os.Getenv("OPENAI_API_KEY")
+	if apiKey == "" {
+		t.Skip("Skipping integration test: OPENAI_API_KEY not set")
+	}
+	baseURL := os.Getenv("OPENAI_BASE_URL")
+	model := os.Getenv("OPENAI_MODEL")
+	if model == "" {
+		model = "gpt-3.5-turbo"
+	}
+
+	// Input with newlines
+	input := "Line 1\nLine 2\nLine 3"
+
+	translator := NewLLMTranslator(apiKey, baseURL, model)
+
+	// 1. Test Single Translate
+	t.Run("Single Translate", func(t *testing.T) {
+		result, err := translator.Translate(context.Background(), input, "en", "zh-CN")
+		require.NoError(t, err)
+
+		// Check if newlines are preserved
+		assert.Contains(t, result, "\n", "Result should contain newlines")
+		lines := strings.Split(result, "\n")
+		// We expect roughly the same number of lines
+		assert.GreaterOrEqual(t, len(lines), 3, "Result should have at least 3 lines")
+		t.Logf("Single Translate Result:\n%s", result)
+	})
+
+	// 2. Test Batch Translate
+	t.Run("Batch Translate", func(t *testing.T) {
+		results, err := translator.TranslateBatch(context.Background(), []string{input}, "en", "zh-CN")
+		require.NoError(t, err)
+		require.Len(t, results, 1)
+
+		// Check if newlines are preserved
+		assert.Contains(t, results[0], "\n", "Result should contain newlines")
+		lines := strings.Split(results[0], "\n")
+		assert.GreaterOrEqual(t, len(lines), 3, "Result should have at least 3 lines")
+		t.Logf("Batch Translate Result:\n%s", results[0])
+	})
+}

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/briandowns/spinner v1.23.2
 	github.com/fatih/color v1.18.0
 	github.com/sashabaranov/go-openai v1.41.2
+	github.com/smacker/go-tree-sitter v0.0.0-20240827094217-dd81d9e9be82
 	github.com/spf13/cobra v1.10.2
 	github.com/spf13/viper v1.21.0
 	github.com/stretchr/testify v1.11.1
@@ -21,7 +22,6 @@ require (
 	github.com/pelletier/go-toml/v2 v2.2.4 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/sagikazarmark/locafero v0.12.0 // indirect
-	github.com/smacker/go-tree-sitter v0.0.0-20240827094217-dd81d9e9be82 // indirect
 	github.com/spf13/afero v1.15.0 // indirect
 	github.com/spf13/cast v1.10.0 // indirect
 	github.com/spf13/pflag v1.0.10 // indirect

tests/rust_integration_test.go

@@ -39,14 +39,15 @@ func TestRustSupport(t *testing.T) {
 	assert.True(t, strings.HasSuffix(result.File, "lib.rs"))
 	assert.NotEmpty(t, result.Comments)
 
-	// Expect 6 comments after merging consecutive lines:
+	// Expect 7 comments after merging consecutive lines:
 	// 1. Merged inner doc (lines 1-2)
 	// 2. Inner doc (line 4)
 	// 3. Outer doc (line 6)
 	// 4. Line comment (line 8)
 	// 5. Block comment (lines 9-10)
 	// 6. Merged line comment (lines 11-12)
-	assert.Equal(t, 6, len(result.Comments), "Should have exactly 6 comments after merging")
+	// 7. Merged doc comment (lines 15-18)
+	assert.Equal(t, 7, len(result.Comments), "Should have exactly 7 comments after merging")
 
 	// Verify Language and Types
 	foundInnerDoc := false
@@ -92,4 +93,13 @@ func TestRustSupport(t *testing.T) {
 		assert.Equal(t, 1, startLine, "First comment should start on line 1")
 		assert.Equal(t, 2, endLine, "First comment (lines 1-2 merged) should end on line 2, not 3")
 	}
+
+	// Verify the last merged doc comment (lines 15-18)
+	if len(result.Comments) >= 7 {
+		lastComment := result.Comments[6]
+		assert.Equal(t, "PrecompileContract", lastComment["symbol"])
+		src := lastComment["sourceText"].(string)
+		assert.Contains(t, src, "A mapping of precompile contracts")
+		assert.Contains(t, src, "dynamic representation")
+	}
 }

tests/testdata/lib.rs

@@ -11,3 +11,12 @@ fn main() {
     // good
     // luck
 }
+
+/// A mapping of precompile contracts that can be either static (builtin) or dynamic.
+///
+/// This is an optimization that allows us to keep using the static precompiles
+/// until we need to modify them, at which point we convert to the dynamic representation.
+struct PrecompileContract {
+    address: u64,
+    name: String,
+}