Fix #5301: Prevent early termination in list-files when directory has many files

src/services/glob/list-files.ts

@@ -26,15 +26,22 @@ export async function listFiles(dirPath: string, recursive: boolean, limit: numb
 	// Get ripgrep path
 	const rgPath = await getRipgrepPath()
 
-	// Get files using ripgrep
-	const files = await listFilesWithRipgrep(rgPath, dirPath, recursive, limit)
-
-	// Get directories with proper filtering
+	// Get directories with proper filtering first to ensure we capture the structure
 	const gitignorePatterns = await parseGitignoreFile(dirPath, recursive)
 	const directories = await listFilteredDirectories(dirPath, recursive, gitignorePatterns)
 
+	// Get files using ripgrep with a higher limit to avoid early termination
+	// We'll use a higher internal limit and then apply balanced sampling
+	const internalLimit = Math.max(limit * 3, 1000) // Use 3x the requested limit or 1000, whichever is higher
+	const files = await listFilesWithRipgrep(rgPath, dirPath, recursive, internalLimit)
+
+	// Apply balanced sampling to ensure fair representation across directories
+	// Reserve some space for directories in the limit
+	const filesLimit = Math.max(limit - directories.length, Math.floor(limit * 0.8))
+	const balancedFiles = applyBalancedSampling(files, directories, filesLimit)
+
 	// Combine and format the results
-	return formatAndCombineResults(files, directories, limit)
+	return formatAndCombineResults(balancedFiles, directories, limit)
 }
 
 /**
@@ -304,6 +311,91 @@ function isIgnoredByGitignore(dirName: string, gitignorePatterns: string[]): boo
 	return false
 }
 
+/**
+ * Apply balanced sampling to ensure fair representation across directories
+ * This prevents one large directory from dominating the file list
+ */
+function applyBalancedSampling(files: string[], directories: string[], limit: number): string[] {
+	if (files.length <= limit) {
+		return files
+	}
+
+	// Group files by their parent directory
+	const filesByDirectory = new Map<string, string[]>()
+
+	for (const file of files) {
+		const dir = path.dirname(file)
+		if (!filesByDirectory.has(dir)) {
+			filesByDirectory.set(dir, [])
+		}
+		filesByDirectory.get(dir)!.push(file)
+	}
+
+	// Improved balanced sampling algorithm
+	const dirEntries = Array.from(filesByDirectory.entries())
+	dirEntries.sort(([a], [b]) => a.localeCompare(b))
+
+	const result: string[] = []
+	const dirCount = dirEntries.length
+
+	// Ensure each directory gets at least a minimum number of files
+	const minFilesPerDir = Math.max(3, Math.floor(limit / (dirCount * 4))) // At least 3 files per dir, or 1/4 of fair share
+	const maxFilesPerDir = Math.floor(limit / dirCount) + 10 // Allow some directories to have more files
+
+	// First pass: give each directory its minimum allocation
+	let remainingLimit = limit
+	const dirAllocations = new Map<string, number>()
+
+	for (const [dir, dirFiles] of dirEntries) {
+		const allocation = Math.min(minFilesPerDir, dirFiles.length, remainingLimit)
+		dirAllocations.set(dir, allocation)
+		remainingLimit -= allocation
+	}
+
+	// Second pass: distribute remaining slots proportionally to directory sizes
+	if (remainingLimit > 0) {
+		const totalFiles = dirEntries.reduce((sum, [, dirFiles]) => sum + dirFiles.length, 0)
+
+		for (const [dir, dirFiles] of dirEntries) {
+			const currentAllocation = dirAllocations.get(dir)!
+			const proportion = dirFiles.length / totalFiles
+			const additionalSlots = Math.min(
+				Math.floor(remainingLimit * proportion),
+				maxFilesPerDir - currentAllocation,
+				dirFiles.length - currentAllocation,
+			)
+
+			if (additionalSlots > 0) {
+				dirAllocations.set(dir, currentAllocation + additionalSlots)
+				remainingLimit -= additionalSlots
+			}
+		}
+	}
+
+	// Third pass: distribute any remaining slots to directories that can take them
+	for (const [dir, dirFiles] of dirEntries) {
+		if (remainingLimit <= 0) break
+
+		const currentAllocation = dirAllocations.get(dir)!
+		const canTakeMore = Math.min(remainingLimit, dirFiles.length - currentAllocation)
+
+		if (canTakeMore > 0) {
+			dirAllocations.set(dir, currentAllocation + canTakeMore)
+			remainingLimit -= canTakeMore
+		}
+	}
+
+	// Collect the files based on allocations
+	for (const [dir, dirFiles] of dirEntries) {
+		const allocation = dirAllocations.get(dir)!
+		dirFiles.sort() // Ensure consistent ordering
+		const selectedFiles = dirFiles.slice(0, allocation)
+		result.push(...selectedFiles)
+	}
+
+	return result
+}
+
 /**
  * Combine file and directory results and format them properly
  */
@@ -338,23 +430,20 @@ async function execRipgrep(rgPath: string, args: string[], limit: number): Promi
 		let output = ""
 		let results: string[] = []
 
-		// Set timeout to avoid hanging
+		// Set timeout to avoid hanging - increased to allow more complete traversal
 		const timeoutId = setTimeout(() => {
 			rgProcess.kill()
 			console.warn("ripgrep timed out, returning partial results")
-			resolve(results.slice(0, limit))
-		}, 10_000)
+			resolve(results) // Don't slice here either
+		}, 15_000)
 
 		// Process stdout data as it comes in
 		rgProcess.stdout.on("data", (data) => {
 			output += data.toString()
 			processRipgrepOutput()
 
-			// Kill the process if we've reached the limit
-			if (results.length >= limit) {
-				rgProcess.kill()
-				clearTimeout(timeoutId) // Clear the timeout when we kill the process due to reaching the limit
-			}
+			// Don't kill the process early - let it complete to get full directory structure
+			// The balanced sampling will be applied later in applyBalancedSampling
 		})
 
 		// Process stderr but don't fail on non-zero exit codes
@@ -375,7 +464,7 @@ async function execRipgrep(rgPath: string, args: string[], limit: number): Promi
 				console.warn(`ripgrep process exited with code ${code}, returning partial results`)
 			}
 
-			resolve(results.slice(0, limit))
+			resolve(results) // Don't slice here - let balanced sampling handle the limit
 		})
 
 		// Handle process errors
@@ -396,12 +485,10 @@ async function execRipgrep(rgPath: string, args: string[], limit: number): Promi
 				output = ""
 			}
 
-			// Process each complete line
+			// Process each complete line - don't limit here, let balanced sampling handle it
 			for (const line of lines) {
-				if (line.trim() && results.length < limit) {
+				if (line.trim()) {
 					results.push(line)
-				} else if (results.length >= limit) {
-					break
 				}
 			}
 		}

test-issue-5301/a/file001.txt

@@ -0,0 +1 @@
+Content of file 1

test-issue-5301/a/file002.txt

@@ -0,0 +1 @@
+Content of file 2

test-issue-5301/a/file003.txt

@@ -0,0 +1 @@
+Content of file 3

test-issue-5301/a/file004.txt

@@ -0,0 +1 @@
+Content of file 4

test-issue-5301/a/file005.txt

@@ -0,0 +1 @@
+Content of file 5

test-issue-5301/a/file006.txt

@@ -0,0 +1 @@
+Content of file 6

test-issue-5301/a/file007.txt

@@ -0,0 +1 @@
+Content of file 7

test-issue-5301/a/file008.txt

@@ -0,0 +1 @@
+Content of file 8

test-issue-5301/a/file009.txt

@@ -0,0 +1 @@
+Content of file 9

test-issue-5301/a/file010.txt

@@ -0,0 +1 @@
+Content of file 10

test-issue-5301/a/file011.txt

@@ -0,0 +1 @@
+Content of file 11

test-issue-5301/a/file012.txt

@@ -0,0 +1 @@
+Content of file 12

test-issue-5301/a/file013.txt

@@ -0,0 +1 @@
+Content of file 13

test-issue-5301/a/file014.txt

@@ -0,0 +1 @@
+Content of file 14

test-issue-5301/a/file015.txt

@@ -0,0 +1 @@
+Content of file 15

test-issue-5301/a/file016.txt

@@ -0,0 +1 @@
+Content of file 16

test-issue-5301/a/file017.txt

@@ -0,0 +1 @@
+Content of file 17

test-issue-5301/a/file018.txt

@@ -0,0 +1 @@
+Content of file 18

test-issue-5301/a/file019.txt

@@ -0,0 +1 @@
+Content of file 19

test-issue-5301/a/file020.txt

@@ -0,0 +1 @@
+Content of file 20

test-issue-5301/a/file021.txt

@@ -0,0 +1 @@
+Content of file 21

test-issue-5301/a/file022.txt

@@ -0,0 +1 @@
+Content of file 22

test-issue-5301/a/file023.txt

@@ -0,0 +1 @@
+Content of file 23

test-issue-5301/a/file024.txt

@@ -0,0 +1 @@
+Content of file 24

test-issue-5301/a/file025.txt

@@ -0,0 +1 @@
+Content of file 25

test-issue-5301/a/file026.txt

@@ -0,0 +1 @@
+Content of file 26

test-issue-5301/a/file027.txt

@@ -0,0 +1 @@
+Content of file 27

test-issue-5301/a/file028.txt

@@ -0,0 +1 @@
+Content of file 28

test-issue-5301/a/file029.txt

@@ -0,0 +1 @@
+Content of file 29

test-issue-5301/a/file030.txt

@@ -0,0 +1 @@
+Content of file 30

test-issue-5301/a/file031.txt

@@ -0,0 +1 @@
+Content of file 31

test-issue-5301/a/file032.txt

@@ -0,0 +1 @@
+Content of file 32

test-issue-5301/a/file033.txt

@@ -0,0 +1 @@
+Content of file 33

test-issue-5301/a/file034.txt

@@ -0,0 +1 @@
+Content of file 34

test-issue-5301/a/file035.txt

@@ -0,0 +1 @@
+Content of file 35

test-issue-5301/a/file036.txt

@@ -0,0 +1 @@
+Content of file 36

test-issue-5301/a/file037.txt

@@ -0,0 +1 @@
+Content of file 37

test-issue-5301/a/file038.txt

@@ -0,0 +1 @@
+Content of file 38

test-issue-5301/a/file039.txt

@@ -0,0 +1 @@
+Content of file 39

test-issue-5301/a/file040.txt

@@ -0,0 +1 @@
+Content of file 40

test-issue-5301/a/file041.txt

@@ -0,0 +1 @@
+Content of file 41

test-issue-5301/a/file042.txt

@@ -0,0 +1 @@
+Content of file 42

test-issue-5301/a/file043.txt

@@ -0,0 +1 @@
+Content of file 43

test-issue-5301/a/file044.txt

@@ -0,0 +1 @@
+Content of file 44

test-issue-5301/a/file045.txt

@@ -0,0 +1 @@
+Content of file 45

test-issue-5301/a/file046.txt

@@ -0,0 +1 @@
+Content of file 46

test-issue-5301/a/file047.txt

@@ -0,0 +1 @@
+Content of file 47

test-issue-5301/a/file048.txt

@@ -0,0 +1 @@
+Content of file 48

test-issue-5301/a/file049.txt

@@ -0,0 +1 @@
+Content of file 49

test-issue-5301/a/file050.txt

@@ -0,0 +1 @@
+Content of file 50

test-issue-5301/a/file051.txt

@@ -0,0 +1 @@
+Content of file 51

test-issue-5301/a/file052.txt

@@ -0,0 +1 @@
+Content of file 52

test-issue-5301/a/file053.txt

@@ -0,0 +1 @@
+Content of file 53

test-issue-5301/a/file054.txt

@@ -0,0 +1 @@
+Content of file 54

test-issue-5301/a/file055.txt

@@ -0,0 +1 @@
+Content of file 55

test-issue-5301/a/file056.txt

@@ -0,0 +1 @@
+Content of file 56

test-issue-5301/a/file057.txt

@@ -0,0 +1 @@
+Content of file 57

test-issue-5301/a/file058.txt

@@ -0,0 +1 @@
+Content of file 58

test-issue-5301/a/file059.txt

@@ -0,0 +1 @@
+Content of file 59

test-issue-5301/a/file060.txt

@@ -0,0 +1 @@
+Content of file 60

test-issue-5301/a/file061.txt

@@ -0,0 +1 @@
+Content of file 61

test-issue-5301/a/file062.txt

@@ -0,0 +1 @@
+Content of file 62

test-issue-5301/a/file063.txt

@@ -0,0 +1 @@
+Content of file 63

test-issue-5301/a/file064.txt

@@ -0,0 +1 @@
+Content of file 64

test-issue-5301/a/file065.txt

@@ -0,0 +1 @@
+Content of file 65

test-issue-5301/a/file066.txt

@@ -0,0 +1 @@
+Content of file 66

test-issue-5301/a/file067.txt

@@ -0,0 +1 @@
+Content of file 67

test-issue-5301/a/file068.txt

@@ -0,0 +1 @@
+Content of file 68

test-issue-5301/a/file069.txt

@@ -0,0 +1 @@
+Content of file 69

test-issue-5301/a/file070.txt

@@ -0,0 +1 @@
+Content of file 70

test-issue-5301/a/file071.txt

@@ -0,0 +1 @@
+Content of file 71

test-issue-5301/a/file072.txt

@@ -0,0 +1 @@
+Content of file 72

test-issue-5301/a/file073.txt

@@ -0,0 +1 @@
+Content of file 73

test-issue-5301/a/file074.txt

@@ -0,0 +1 @@
+Content of file 74

test-issue-5301/a/file075.txt

@@ -0,0 +1 @@
+Content of file 75