Fix files with very long paths always being treated as unindexed

* bdx/index.py (SymbolIndex.all_files): Rework to yield all paths
properly even if their db terms have been truncated due to length
limits.
* tests/test_index.py (test_indexing_very_long_path): New test.

Author: mkcms

bdx/index.py

@@ -762,10 +762,26 @@ def delete_file(self, file: Path):
 
     def all_files(self) -> Iterator[Path]:
         """Yield all the files indexed in this SymbolIndex."""
+        seen_paths = set()
+
         for value in self.iter_prefix("path", ""):
-            path = Path(value)
-            if path.is_absolute():
-                yield path
+            # Check if the path stored in database has been truncated
+            # because it's length exceeded MAX_TERM_SIZE.  If so, then
+            # we need to search this term by wildcard and return all
+            # paths that are actually stored in the data of each
+            # document.
+            raw_term = self.schema["path"].prefix.encode() + value.encode()
+            if len(raw_term) == MAX_TERM_SIZE:
+                query = f"path:{value}*"
+                results = self.search(query)
+                paths = set([x.path for x in results]).difference(seen_paths)
+                seen_paths.update(paths)
+                yield from paths
+            else:
+                path = Path(value)
+                if path.is_absolute() and path not in seen_paths:
+                    seen_paths.add(path)
+                    yield path
 
     def iter_prefix(self, field: str, value_prefix: str) -> Iterator[str]:
         """Return all the possible values for ``field`` with given prefix."""

tests/test_index.py

@@ -9,6 +9,7 @@
 # isort: off
 from bdx.binary import Exclusion, SymbolType
 from bdx.index import (
+    MAX_TERM_SIZE,
     IndexingOptions,
     SymbolIndex,
     SymbolNameField,
@@ -462,6 +463,27 @@ def test_indexing_not_triggered_if_mtime_not_changed(tmp_path):
         assert "quux" not in symbols
 
 
+def test_indexing_very_long_path(tmp_path):
+    index_path = tmp_path / "index"
+    dir = tmp_path / "build"
+    dir.mkdir()
+
+    filename = "0" * (os.pathconf(tmp_path, "PC_NAME_MAX") - 2) + ".o"
+    file: Path = dir / filename
+    short_file: Path = dir / "short.o"
+
+    assert (
+        len(str(file)) + len(SymbolIndex.SCHEMA["path"].prefix) > MAX_TERM_SIZE
+    )
+
+    _compile_file(file, "void foo() {}", ["-c"])
+    _compile_file(short_file, "void shortfile() {}", ["-c"])
+    index_binary_directory(dir, index_path, IndexingOptions())
+    with SymbolIndex.open(index_path, readonly=True) as index:
+        all_files = sorted(index.all_files())
+        assert all_files == [file, short_file]
+
+
 def test_searching_by_wildcard(readonly_index):
     symbols = set(readonly_index.search("name:a_*"))
     assert symbols