[ty] Add some completion ranking improvements (#20807)

Co-authored-by: Micha Reiser <[email protected]>
Co-authored-by: Alex Waygood <[email protected]>

Author: 3 people

.github/workflows/ci.yaml

@@ -721,7 +721,7 @@ jobs:
       - name: "Install Rust toolchain"
         run: rustup show
       - name: "Run ty completion evaluation"
-        run: cargo run --release --package ty_completion_eval -- all --threshold 0.1 --tasks /tmp/completion-evaluation-tasks.csv
+        run: cargo run --release --package ty_completion_eval -- all --threshold 0.4 --tasks /tmp/completion-evaluation-tasks.csv
       - name: "Ensure there are no changes"
         run: diff ./crates/ty_completion_eval/completion-evaluation-tasks.csv /tmp/completion-evaluation-tasks.csv
 

crates/ty_completion_eval/README.md

@@ -7,7 +7,7 @@ To run a full evaluation, run the `ty_completion_eval` crate with the
 `all` command from the root of this repository:
 
 ```console
-cargo run --release --package ty_completion_eval -- all
+cargo run --profile profiling --package ty_completion_eval -- all
 ```
 
 The output should look like this:
@@ -24,7 +24,7 @@ you can ask the evaluation to write CSV data that contains the rank of
 the expected answer in each completion request:
 
 ```console
-cargo r -r -p ty_completion_eval -- all --tasks ./crates/ty_completion_eval/completion-evaluation-tasks.csv
+cargo r --profile profiling -p ty_completion_eval -- all --tasks ./crates/ty_completion_eval/completion-evaluation-tasks.csv
 ```
 
 To debug a _specific_ task and look at the actual results, use the `show-one`
@@ -133,7 +133,7 @@ CI will also fail if the individual task results have changed.
 To make CI pass, you can just re-run the evaluation locally and commit the results:
 
 ```console
-cargo r -r -p ty_completion_eval -- all --tasks ./crates/ty_completion_eval/completion-evaluation-tasks.csv
+cargo r --profile profiling -p ty_completion_eval -- all --tasks ./crates/ty_completion_eval/completion-evaluation-tasks.csv
 ```
 
 CI fails in this case because it would be best to scrutinize the differences here.

crates/ty_completion_eval/completion-evaluation-tasks.csv

@@ -1,17 +1,20 @@
 name,file,index,rank
+fstring-completions,main.py,0,1
 higher-level-symbols-preferred,main.py,0,
 higher-level-symbols-preferred,main.py,1,1
-import-deprioritizes-dunder,main.py,0,195
-import-deprioritizes-sunder,main.py,0,195
-internal-typeshed-hidden,main.py,0,43
+import-deprioritizes-dunder,main.py,0,1
+import-deprioritizes-sunder,main.py,0,1
+internal-typeshed-hidden,main.py,0,4
+none-completion,main.py,0,11
 numpy-array,main.py,0,
-numpy-array,main.py,1,32
-object-attr-instance-methods,main.py,0,7
+numpy-array,main.py,1,1
+object-attr-instance-methods,main.py,0,1
 object-attr-instance-methods,main.py,1,1
-raise-uses-base-exception,main.py,0,42
-scope-existing-over-new-import,main.py,0,495
-scope-prioritize-closer,main.py,0,152
-scope-simple-long-identifier,main.py,0,140
-ty-extensions-lower-stdlib,main.py,0,142
-type-var-typing-over-ast,main.py,0,65
-type-var-typing-over-ast,main.py,1,353
+raise-uses-base-exception,main.py,0,2
+scope-existing-over-new-import,main.py,0,474
+scope-prioritize-closer,main.py,0,2
+scope-simple-long-identifier,main.py,0,1
+tstring-completions,main.py,0,1
+ty-extensions-lower-stdlib,main.py,0,7
+type-var-typing-over-ast,main.py,0,3
+type-var-typing-over-ast,main.py,1,270

crates/ty_completion_eval/src/main.rs

@@ -15,6 +15,9 @@ use regex::bytes::Regex;
 use ruff_db::files::system_path_to_file;
 use ruff_db::system::{OsSystem, SystemPath, SystemPathBuf};
 use ty_ide::Completion;
+use ty_project::metadata::Options;
+use ty_project::metadata::options::EnvironmentOptions;
+use ty_project::metadata::value::RelativePathBuf;
 use ty_project::{ProjectDatabase, ProjectMetadata};
 use ty_python_semantic::ModuleName;
 
@@ -117,8 +120,8 @@ impl ShowOneCommand {
             && self
                 .file_name
                 .as_ref()
-                .is_some_and(|name| name == task.cursor_name())
-            && self.index.is_some_and(|index| index == task.cursor.index)
+                .is_none_or(|name| name == task.cursor_name())
+            && self.index.is_none_or(|index| index == task.cursor.index)
     }
 }
 
@@ -278,6 +281,14 @@ impl Task {
 
         let system = OsSystem::new(project_path);
         let mut project_metadata = ProjectMetadata::discover(project_path, &system)?;
+        // Explicitly point ty to the .venv to avoid any set VIRTUAL_ENV variable to take precedence.
+        project_metadata.apply_options(Options {
+            environment: Some(EnvironmentOptions {
+                python: Some(RelativePathBuf::cli(".venv")),
+                ..EnvironmentOptions::default()
+            }),
+            ..Options::default()
+        });
         project_metadata.apply_configuration_files(&system)?;
         let db = ProjectDatabase::new(project_metadata, system)?;
         Ok(Task {

crates/ty_completion_eval/truth/fstring-completions/completion.toml

@@ -0,0 +1,2 @@
+[settings]
+auto-import = false

crates/ty_completion_eval/truth/fstring-completions/main.py

@@ -0,0 +1,3 @@
+zqzqzq_identifier = 1
+
+print(f"{zqzqzq_<CURSOR: zqzqzq_identifier>}")

crates/ty_completion_eval/truth/fstring-completions/pyproject.toml

@@ -0,0 +1,5 @@
+[project]
+name = "test"
+version = "0.1.0"
+requires-python = ">=3.13"
+dependencies = []

crates/ty_completion_eval/truth/fstring-completions/uv.lock

@@ -0,0 +1,2 @@
+[settings]
+auto-import = false

crates/ty_completion_eval/truth/none-completion/completion.toml

@@ -0,0 +1 @@
+x = Non<CURSOR: None>