Tests: File Indexing and Search Functionality (#1503)

* Add improvements to code indexing and search operations

This commit includes additions for the creation of files, directories, and unit tests related to file indexing and searching operations.

* Shortened lines

* Added doc for TempFolderManager

Author: tom-ludwig

CodeEdit.xcodeproj/project.pbxproj

@@ -265,6 +265,10 @@
 		611192062B08CCF600D4459B /* SearchIndexer+Add.swift in Sources */ = {isa = PBXBuildFile; fileRef = 611192052B08CCF600D4459B /* SearchIndexer+Add.swift */; };
 		611192082B08CCFD00D4459B /* SearchIndexer+Terms.swift in Sources */ = {isa = PBXBuildFile; fileRef = 611192072B08CCFD00D4459B /* SearchIndexer+Terms.swift */; };
 		6111920C2B08CD0B00D4459B /* SearchIndexer+InternalMethods.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6111920B2B08CD0B00D4459B /* SearchIndexer+InternalMethods.swift */; };
+		6130535C2B23933D00D767E3 /* MemoryIndexingTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6130535B2B23933D00D767E3 /* MemoryIndexingTests.swift */; };
+		6130535F2B23A31300D767E3 /* MemorySearchTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6130535E2B23A31300D767E3 /* MemorySearchTests.swift */; };
+		613053652B23A49300D767E3 /* TemporaryFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 613053642B23A49300D767E3 /* TemporaryFile.swift */; };
+		6130536B2B24722C00D767E3 /* AsyncIndexingTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6130536A2B24722C00D767E3 /* AsyncIndexingTests.swift */; };
 		613DF55E2B08DD5D00E9D902 /* FileHelper.swift in Sources */ = {isa = PBXBuildFile; fileRef = 613DF55D2B08DD5D00E9D902 /* FileHelper.swift */; };
 		61538B902B111FE800A88846 /* String+AppearancesOfSubstring.swift in Sources */ = {isa = PBXBuildFile; fileRef = 61538B8F2B111FE800A88846 /* String+AppearancesOfSubstring.swift */; };
 		61538B932B11201900A88846 /* String+Character.swift in Sources */ = {isa = PBXBuildFile; fileRef = 61538B922B11201900A88846 /* String+Character.swift */; };
@@ -787,6 +791,10 @@
 		611192052B08CCF600D4459B /* SearchIndexer+Add.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "SearchIndexer+Add.swift"; sourceTree = "<group>"; };
 		611192072B08CCFD00D4459B /* SearchIndexer+Terms.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "SearchIndexer+Terms.swift"; sourceTree = "<group>"; };
 		6111920B2B08CD0B00D4459B /* SearchIndexer+InternalMethods.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "SearchIndexer+InternalMethods.swift"; sourceTree = "<group>"; };
+		6130535B2B23933D00D767E3 /* MemoryIndexingTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MemoryIndexingTests.swift; sourceTree = "<group>"; };
+		6130535E2B23A31300D767E3 /* MemorySearchTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MemorySearchTests.swift; sourceTree = "<group>"; };
+		613053642B23A49300D767E3 /* TemporaryFile.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TemporaryFile.swift; sourceTree = "<group>"; };
+		6130536A2B24722C00D767E3 /* AsyncIndexingTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AsyncIndexingTests.swift; sourceTree = "<group>"; };
 		613DF55D2B08DD5D00E9D902 /* FileHelper.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FileHelper.swift; sourceTree = "<group>"; };
 		61538B8F2B111FE800A88846 /* String+AppearancesOfSubstring.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "String+AppearancesOfSubstring.swift"; sourceTree = "<group>"; };
 		61538B922B11201900A88846 /* String+Character.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "String+Character.swift"; sourceTree = "<group>"; };
@@ -1245,6 +1253,7 @@
 			children = (
 				4EE96ECC296059D200FFBEA8 /* Mocks */,
 				4EE96ECA2960565E00FFBEA8 /* DocumentsUnitTests.swift */,
+				613053582B23916D00D767E3 /* Indexer */,
 			);
 			path = Documents;
 			sourceTree = "<group>";
@@ -2206,6 +2215,17 @@
 			path = Indexer;
 			sourceTree = "<group>";
 		};
+		613053582B23916D00D767E3 /* Indexer */ = {
+			isa = PBXGroup;
+			children = (
+				6130535B2B23933D00D767E3 /* MemoryIndexingTests.swift */,
+				6130535E2B23A31300D767E3 /* MemorySearchTests.swift */,
+				6130536A2B24722C00D767E3 /* AsyncIndexingTests.swift */,
+				613053642B23A49300D767E3 /* TemporaryFile.swift */,
+			);
+			path = Indexer;
+			sourceTree = "<group>";
+		};
 		6C092EDC2A53A63E00489202 /* Views */ = {
 			isa = PBXGroup;
 			children = (
@@ -3483,11 +3503,15 @@
 			buildActionMask = 2147483647;
 			files = (
 				583E528C29361B39001AB554 /* CodeEditUITests.swift in Sources */,
+				613053652B23A49300D767E3 /* TemporaryFile.swift in Sources */,
 				587B60F82934124200D5CD8F /* CEWorkspaceFileManagerTests.swift in Sources */,
+				6130535F2B23A31300D767E3 /* MemorySearchTests.swift in Sources */,
 				587B61012934170A00D5CD8F /* UnitTests_Extensions.swift in Sources */,
 				283BDCC52972F236002AFF81 /* AcknowledgementsTests.swift in Sources */,
 				4EE96ECB2960565E00FFBEA8 /* DocumentsUnitTests.swift in Sources */,
 				4EE96ECE296059E000FFBEA8 /* NSHapticFeedbackPerformerMock.swift in Sources */,
+				6130535C2B23933D00D767E3 /* MemoryIndexingTests.swift in Sources */,
+				6130536B2B24722C00D767E3 /* AsyncIndexingTests.swift in Sources */,
 				587B612E293419B700D5CD8F /* CodeFileTests.swift in Sources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;

CodeEdit/Features/Documents/Indexer/SearchIndexer+AsyncController.swift

@@ -35,6 +35,32 @@ extension SearchIndexer {
         }
 
         // MARK: - Search
+
+        /// Performs an asynchronous progressive search on the index for the specified query.
+        ///
+        /// - Parameters:
+        ///   - query: The search query string.
+        ///   - maxResults: The maximum number of results to retrieve in each chunk.
+        ///   - timeout: The timeout duration for each search operation. Default is 1.0 second.
+        ///
+        /// - Returns: An asynchronous stream (`AsyncStream`) of search results in chunks.
+        /// The search results are returned in the form of a `SearchIndexer.ProgressivSearch.Results` object.
+        ///
+        /// This function initiates a progressive search on the index for the specified query
+        /// and asynchronously yields search results in chunks using an `AsyncStream`.
+        /// The search continues until there are no more results or the specified timeout is reached.
+        ///
+        /// - Warning: Prior to calling this function,
+        /// ensure that the `index` has been flushed to search within the most up-to-date data.
+        ///
+        /// Example usage:
+        /// ```swift
+        /// let searchStream = await asyncController.search(query: searchQuery, 20)
+        /// for try await result in searchStream {
+        ///     // Process each result
+        ///     print(result)
+        /// }
+        /// ```
         func search(
             query: String,
             _ maxResults: Int,
@@ -55,16 +81,26 @@ extension SearchIndexer {
 
         // MARK: - Add
 
+        /// Adds files from an array of TextFile objects to the index asynchronously.
+        ///
+        /// - Parameters:
+        ///   - files: An array of TextFile objects containing the information about the files to be added.
+        ///   - flushWhenComplete: A boolean flag indicating whether to flush 
+        ///   the index when the operation is complete. Default is `false`.
+        ///
+        /// - Returns: An array of booleans indicating the success of adding each file to the index.
         func addText(
             files: [TextFile],
             flushWhenComplete: Bool = false
         ) async -> [Bool] {
 
             var addedFiles = [Bool]()
 
+            // Asynchronously iterate through the provided files using a task group
             await withTaskGroup(of: Bool.self) { taskGroup in
                 for file in files {
                     taskGroup.addTask {
+                        // Add the file to the index and return the success status
                         return self.index.addFileWithText(file.url, text: file.text, canReplace: true)
                     }
                 }
@@ -73,12 +109,24 @@ extension SearchIndexer {
                     addedFiles.append(result)
                 }
             }
+
             if flushWhenComplete {
                 index.flush()
             }
+
             return addedFiles
         }
 
+        /// Adds files from an array of URLs to the index asynchronously.
+        ///
+        /// - Parameters:
+        ///   - urls: An array of URLs representing the file locations to be added to the index.
+        ///   - flushWhenComplete: A boolean flag indicating whether to flush
+        ///   the index when the operation is complete. Default is `false`.
+        ///
+        /// - Returns: An array of booleans indicating the success of adding each file to the index.
+        /// - Warning: Prefer using `addText` when possible as SearchKit does not have the ability
+        ///  to read every file type. For example, it is often not possible to read Swift files.
         func addFiles(
             urls: [URL],
             flushWhenComplete: Bool = false
@@ -100,6 +148,16 @@ extension SearchIndexer {
             return addedURLs
         }
 
+        /// Adds files from a folder specified by the given URL to the index asynchronously.
+        ///
+        /// - Parameters:
+        ///   - url: The URL of the folder containing files to be added to the index.
+        ///   - flushWhenComplete: A boolean flag indicating whether to flush 
+        ///   the index when the operation is complete. Default is `false`.
+        ///
+        /// This function uses asynchronous processing to add files from the specified folder to the index.
+        ///
+        /// - Note: Subfolders within the specified folder are also processed.
         func addFolder(
             url: URL,
             flushWhenComplete: Bool = false

CodeEditTests/Features/Documents/Indexer/AsyncIndexingTests.swift

@@ -0,0 +1,76 @@
+//
+//  AsyncIndexingTests.swift
+//  CodeEditTests
+//
+//  Created by Tommy Ludwig on 09.12.23.
+//
+
+import XCTest
+@testable import CodeEdit
+
+final class AsyncIndexingTests: XCTestCase {
+    func testAddDocuments() {
+        guard let index = SearchIndexer.Memory.create() else {
+            XCTFail("Failed to create an index")
+            return
+        }
+
+        let asyncManager = SearchIndexer.AsyncManager(index: index)
+        let expectation = XCTestExpectation(description: "Async operations completed")
+        let tempFile1 = TemporaryFile().url
+        let tempFile2 = TemporaryFile().url
+
+        Task {
+            let results = await asyncManager.addFiles(urls: [tempFile1, tempFile2])
+            XCTAssertEqual(results.count, 2, "Unexpected indexing results.")
+            asyncManager.index.flush()
+            let documents = asyncManager.index.documents()
+            XCTAssertEqual(documents.count, 2)
+            expectation.fulfill()
+        }
+
+        wait(for: [expectation], timeout: 2)
+    }
+
+    func testSearchDocuments() {
+        guard let index = SearchIndexer.Memory.create() else {
+            XCTFail("Failed to create an index")
+            return
+        }
+
+        let asyncManager = SearchIndexer.AsyncManager(index: index)
+        let expectation = XCTestExpectation(description: "Async operations completed")
+
+        let tempFile1 = SearchIndexer.AsyncManager.TextFile(
+            url: TemporaryFile().url,
+            text: "Itaque ratione asperiores."
+        )
+        let tempFile2 = SearchIndexer.AsyncManager.TextFile(
+            url: TemporaryFile().url,
+            text: "Perspiciatis perspiciatis rerum ex asperiores."
+        )
+
+        Task {
+            var searchResults = [URL]()
+            let results = await asyncManager.addText(files: [tempFile1, tempFile2])
+            XCTAssertEqual(results.count, 2, "Unexpected indexing results.")
+            asyncManager.index.flush()
+            let searchStream = await asyncManager.search(query: "asperiores", 10)
+            for try await result in searchStream {
+                let urls: [(URL, Float)] = result.results.compactMap {
+                    ($0.url, $0.score)
+                }
+
+                for (url, _) in urls {
+                    searchResults.append(url)
+                }
+            }
+
+            XCTAssertEqual(searchResults.count, 2)
+            expectation.fulfill()
+        }
+
+        wait(for: [expectation], timeout: 2)
+    }
+
+}

CodeEditTests/Features/Documents/Indexer/MemoryIndexingTests.swift

@@ -0,0 +1,147 @@
+//
+//  MemoryIndexing.swift
+//  CodeEditTests
+//
+//  Created by Tommy Ludwig on 08.12.23.
+//
+
+import XCTest
+@testable import CodeEdit
+
+final class MemoryIndexingTests: XCTestCase {
+    func testIndexFile() {
+        guard let index = SearchIndexer.Memory.create() else {
+            XCTFail("Failed to create an index")
+            return
+        }
+
+        let filePath = TemporaryFile().url
+
+        let indexResults = index.addFileWithText(filePath, text: "Hello, World!")
+        XCTAssert(indexResults)
+    }
+
+    func testIndexFiles() {
+        guard let index = SearchIndexer.Memory.create() else {
+            XCTFail("Failed to create an index")
+            return
+        }
+
+        let document1 = TemporaryFile().url
+        let document2 = TemporaryFile().url
+
+        var indexResults = index.addFileWithText(document1, text: "fileContent")
+        XCTAssert(indexResults)
+        indexResults = index.addFileWithText(document2, text: "")
+        XCTAssert(indexResults)
+        index.flush()
+        let res = index.cleanUp()
+        XCTAssertEqual(res, 1)
+    }
+
+    func testIndexFolder() {
+        guard let index = SearchIndexer.Memory.create() else {
+            XCTFail("Failed to create an index")
+            return
+        }
+
+        let folder = TempFolderManager()
+        folder.createCustomFolder()
+        folder.createFiles()
+
+        let indexResults = index.addFolderContent(folderURL: folder.customFolderURL)
+        XCTAssertEqual(indexResults.count, 2)
+
+        index.flush()
+
+        let searchResults = index.search("file")
+        XCTAssertEqual(searchResults.count, 2, "Unexpected search results")
+    }
+
+    func testIndexCleanUp() {
+        guard let index = SearchIndexer.Memory.create() else {
+            XCTFail("Failed to create an index")
+            return
+        }
+
+        let document1 = TemporaryFile().url
+        let document2 = TemporaryFile().url
+
+        var indexResults = index.addFileWithText(document1, text: "fileContent")
+        XCTAssert(indexResults)
+        indexResults = index.addFileWithText(document2, text: "")
+        XCTAssert(indexResults)
+        index.flush()
+        let res = index.cleanUp()
+        XCTAssertEqual(res, 1)
+    }
+
+    func testCloseIndex() {
+        guard let index = SearchIndexer.Memory.create() else {
+            XCTFail("Failed to create an index")
+            return
+        }
+
+        let filePath = TemporaryFile().url
+
+        let indexResults = index.addFileWithText(filePath, text: "Hello, World!")
+        XCTAssert(indexResults)
+
+        index.close()
+
+        let closedIndexResults = index.addFileWithText(filePath, text: "Hello, World")
+        XCTAssertEqual(closedIndexResults, false)
+    }
+
+    func testDocumentIsIndex() {
+        guard let index = SearchIndexer.Memory.create() else {
+            XCTFail("Failed to create an index")
+            return
+        }
+
+        let filePath = TemporaryFile().url
+
+        let indexResults = index.addFileWithText(filePath, text: "Hello, World!")
+        XCTAssert(indexResults)
+
+        let isIndexed = index.documentIndexed(filePath)
+        XCTAssertEqual(isIndexed, false)
+
+        index.flush()
+        let isIndexedAfterFlush = index.documentIndexed(filePath)
+        XCTAssert(isIndexedAfterFlush)
+    }
+
+    func testSaveAndLoad() {
+        guard let index = SearchIndexer.Memory.create() else {
+            XCTFail("Failed to create an index")
+            return
+        }
+
+        let textFilePath = TemporaryFile().url
+
+        XCTAssertTrue(index.addFileWithText(textFilePath, text: "Illum assumenda iure earum dolorum fugit."))
+
+        index.flush()
+
+        let searchResults = index.search("earum")
+        XCTAssertEqual(1, searchResults.count)
+        XCTAssertEqual(searchResults[0].url, textFilePath)
+
+        // Save the current index.
+        let savedIndex = index.getAsData()
+        XCTAssertNotNil(savedIndex, "Failed to save the index.")
+
+        // Close the index, i.e. the index gets deallocated form memory.
+        index.close()
+
+        // Load the saved index
+        guard let loadedIndex = SearchIndexer.Memory(data: savedIndex!) else {
+            XCTFail("Failed to create an index")
+            return
+        }
+
+        let savedIndexResult = loadedIndex.search("earum")
+        XCTAssertEqual(savedIndexResult.count, 1)
+    }
+}