Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/services/code-index/interfaces/file-processor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ export interface IDirectoryScanner {
onError?: (error: Error) => void,
onBlocksIndexed?: (indexedCount: number) => void,
onFileParsed?: (fileBlockCount: number) => void,
onProgressUpdate?: (message: string) => void,
): Promise<{
codeBlocks: CodeBlock[]
stats: {
Expand Down
47 changes: 45 additions & 2 deletions src/services/code-index/orchestrator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,7 @@ export class CodeIndexOrchestrator {
let cumulativeBlocksIndexed = 0
let cumulativeBlocksFoundSoFar = 0
let batchErrors: Error[] = []
let scanStartTime = Date.now()

const handleFileParsed = (fileBlockCount: number) => {
cumulativeBlocksFoundSoFar += fileBlockCount
Expand All @@ -138,6 +139,13 @@ export class CodeIndexOrchestrator {
this.stateManager.reportBlockIndexingProgress(cumulativeBlocksIndexed, cumulativeBlocksFoundSoFar)
}

// Report initial scanning progress
this.stateManager.setSystemState("Indexing", "Discovering files to scan...")

const handleProgressUpdate = (message: string) => {
this.stateManager.setSystemState("Indexing", message)
}

const result = await this.scanner.scanDirectory(
this.workspacePath,
(batchError: Error) => {
Expand All @@ -149,13 +157,32 @@ export class CodeIndexOrchestrator {
},
handleBlocksIndexed,
handleFileParsed,
handleProgressUpdate,
)

if (!result) {
throw new Error("Scan failed, is scanner initialized?")
}

const { stats } = result
const scanDuration = Date.now() - scanStartTime

// Validate that actual scanning work was performed
const totalFilesProcessed = stats.processed + stats.skipped
const hasActualWork = totalFilesProcessed > 0

// If no files were found to process, this might indicate a configuration issue
if (!hasActualWork) {
this.stateManager.setSystemState("Indexed", "No supported files found in workspace. Index is empty.")
await this._startWatcher()
return
}

// Report scanning completion with meaningful progress
this.stateManager.setSystemState(
"Indexing",
`Scanned ${totalFilesProcessed} files (${stats.processed} processed, ${stats.skipped} cached). Processing code blocks...`,
)

// Check if any blocks were actually indexed successfully
// If no blocks were indexed but blocks were found, it means all batches failed
Expand All @@ -172,7 +199,10 @@ export class CodeIndexOrchestrator {
}

// Check for partial failures - if a significant portion of blocks failed
const failureRate = (cumulativeBlocksFoundSoFar - cumulativeBlocksIndexed) / cumulativeBlocksFoundSoFar
const failureRate =
cumulativeBlocksFoundSoFar > 0
? (cumulativeBlocksFoundSoFar - cumulativeBlocksIndexed) / cumulativeBlocksFoundSoFar
: 0
if (batchErrors.length > 0 && failureRate > 0.1) {
// More than 10% of blocks failed to index
const firstError = batchErrors[0]
Expand All @@ -196,9 +226,22 @@ export class CodeIndexOrchestrator {
)
}

// Provide meaningful completion message based on what was actually done
let completionMessage: string
if (stats.processed === 0 && stats.skipped > 0) {
// All files were cached - no actual processing needed
completionMessage = `Index up-to-date. All ${stats.skipped} files were already cached (no changes detected).`
} else if (cumulativeBlocksIndexed > 0) {
// Some blocks were actually indexed
completionMessage = `Indexing complete. Processed ${stats.processed} files and indexed ${cumulativeBlocksIndexed} code blocks in ${Math.round(scanDuration / 1000)}s.`
} else {
// No blocks found to index
completionMessage = `Scan complete. No code blocks found in ${totalFilesProcessed} files.`
}

await this._startWatcher()

this.stateManager.setSystemState("Indexed", "File watcher started.")
this.stateManager.setSystemState("Indexed", completionMessage)
} catch (error: any) {
console.error("[CodeIndexOrchestrator] Error during indexing:", error)
TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, {
Expand Down
20 changes: 20 additions & 0 deletions src/services/code-index/processors/scanner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -51,17 +51,23 @@ export class DirectoryScanner implements IDirectoryScanner {
onError?: (error: Error) => void,
onBlocksIndexed?: (indexedCount: number) => void,
onFileParsed?: (fileBlockCount: number) => void,
onProgressUpdate?: (message: string) => void,
): Promise<{ codeBlocks: CodeBlock[]; stats: { processed: number; skipped: number }; totalBlockCount: number }> {
const directoryPath = directory
// Capture workspace context at scan start
const scanWorkspace = getWorkspacePathForContext(directoryPath)

// Report file discovery progress
onProgressUpdate?.("Discovering files in workspace...")

// Get all files recursively (handles .gitignore automatically)
const [allPaths, _] = await listFiles(directoryPath, true, MAX_LIST_FILES_LIMIT)

// Filter out directories (marked with trailing '/')
const filePaths = allPaths.filter((p) => !p.endsWith("/"))

onProgressUpdate?.(`Found ${filePaths.length} files. Applying filters...`)

// Initialize RooIgnoreController if not provided
const ignoreController = new RooIgnoreController(directoryPath)

Expand All @@ -83,6 +89,20 @@ export class DirectoryScanner implements IDirectoryScanner {
return scannerExtensions.includes(ext) && !this.ignoreInstance.ignores(relativeFilePath)
})

onProgressUpdate?.(
`Found ${supportedPaths.length} supported files to scan (filtered from ${filePaths.length} total files).`,
)

// Early return if no files to process
if (supportedPaths.length === 0) {
onProgressUpdate?.("No supported files found to scan.")
return {
codeBlocks: [],
stats: { processed: 0, skipped: 0 },
totalBlockCount: 0,
}
}

// Initialize tracking variables
const processedFiles = new Set<string>()
const codeBlocks: CodeBlock[] = []
Expand Down