scip-syntax: adds strict SCIP symbol parsing and formatting (#63443)

Adds strict and performant symbol parsing/formatting for `scip-syntax`.

Parsing is "zero" allocation when the symbol does not contain escapes.
(Technically it does allocate a Vec to hold the descriptors)

Final benchmark numbers:
```
symbol parsing/parse_v1/10000 [19.158 ms 19.231 ms 19.306 ms]
symbol parsing/parse_v1/100000 [252.10 ms 252.48 ms 252.86 ms]
symbol parsing/parse_v1/1000000 [2.9972 s 3.0033 s 3.0094 s]

symbol parsing/parse_v2/10000 [1.1307 ms 1.1357 ms 1.1413 ms]
symbol parsing/parse_v2/100000 [15.645 ms 15.670 ms 15.697 ms]
symbol parsing/parse_v2/1000000 [191.80 ms 192.11 ms 192.44 ms]
```

## Test plan

Some basic unit tests. Verified manually that it produces the same
symbols as the existing parser for all of chromium.scip

Author: Christoph Hegemann

docker-images/syntax-highlighter/Cargo.Bazel.lock

@@ -61,6 +61,7 @@ tree-sitter-highlight = "0.20.1"
 walkdir = "2"
 path-clean = "1"
 camino = "1.1"
+nom = "7.1.3"
 
 scip = "0.3.2"
 protobuf = "3"

docker-images/syntax-highlighter/Cargo.lock

@@ -28,9 +28,7 @@ WORKSPACE_DEPS = [
 rust_library(
     name = "scip_syntax_lib",
     srcs = glob(
-        [
-            "src/*.rs",
-        ],
+        ["src/**/*.rs"],
         allow_empty = False,
         exclude = ["src/main.rs"],
     ),
@@ -49,7 +47,11 @@ rust_library(
 rust_test(
     name = "unit_test",
     size = "small",
-    srcs = glob(["src/*.rs"]),
+    srcs = glob(
+        ["src/**/*.rs"],
+        allow_empty = False,
+        exclude = ["src/main.rs"],
+    ),
     proc_macro_deps = all_crate_deps(
         proc_macro = True,
     ),
@@ -68,7 +70,11 @@ rust_test(
 rust_test(
     name = "integration_test",
     size = "small",
-    srcs = glob(["tests/*.rs"]),
+    srcs = glob(
+        ["src/**/*.rs"],
+        allow_empty = False,
+        exclude = ["src/main.rs"],
+    ),
     compile_data = glob(
         [
             "testdata/**",

docker-images/syntax-highlighter/Cargo.toml

@@ -6,6 +6,10 @@ edition = "2021"
 [[bin]]
 name = "scip-syntax"
 
+[[bench]]
+name = "symbol_parsing"
+harness = false
+
 [dependencies]
 assert_cmd = "2.0.12"
 predicates = "3.0.4"
@@ -26,10 +30,12 @@ walkdir = { workspace = true }
 path-clean = { workspace = true }
 camino = { workspace = true }
 tree-sitter = { workspace = true }
+nom = { workspace = true }
 
 syntax-analysis = { path = "../syntax-analysis" }
 tree-sitter-all-languages = { path = "../tree-sitter-all-languages" }
 tar = "0.4.40"
 
 [dev-dependencies]
 tempfile="3.10.1"
+criterion = { version = "0.4", features = ["html_reports"] }

docker-images/syntax-highlighter/crates/scip-syntax/BUILD.bazel

@@ -0,0 +1,47 @@
+use camino::Utf8Path;
+use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion};
+use scip_syntax::{io::read_index_from_file, scip_strict};
+
+fn parse_symbols(symbols: &[&str]) {
+    for symbol in symbols {
+        scip::symbol::parse_symbol(symbol).unwrap();
+    }
+}
+
+fn parse_symbols_v2(symbols: &[&str]) {
+    for symbol in symbols {
+        scip_strict::Symbol::parse(&symbol).unwrap();
+    }
+}
+
+fn symbols_from_index(path: &str) -> impl Iterator<Item = String> {
+    let index = read_index_from_file(Utf8Path::new(path))
+    .unwrap();
+    index
+        .documents
+        .into_iter()
+        .flat_map(|document| {
+            document
+                .occurrences
+                .into_iter()
+                .map(|occurrence| occurrence.symbol)
+        })
+}
+
+fn bench_symbol_parsing(c: &mut Criterion) {
+    // let all_symbols: Vec<String> = symbols_from_index("~/work/scip-indices/spring-framework-syntactic.scip").collect();
+    let all_symbols: Vec<String> = symbols_from_index("/Users/creek/work/scip-indices/chromium-1.scip").collect();
+    let mut group = c.benchmark_group("symbol parsing");
+    for n in [10_000, 100_000, 1_000_000] {
+        let symbols: Vec<&str> = all_symbols.iter().take(n).map(|s| s.as_str()).collect();
+        group.bench_with_input(BenchmarkId::new("parse_v1", n), &symbols, |b, syms| {
+            b.iter(|| parse_symbols(syms))
+        });
+        group.bench_with_input(BenchmarkId::new("parse_v2", n), &symbols, |b, syms| {
+            b.iter(|| parse_symbols_v2(syms))
+        });
+    }
+}
+
+criterion_group!(benches, bench_symbol_parsing);
+criterion_main!(benches);

docker-images/syntax-highlighter/crates/scip-syntax/Cargo.toml

@@ -14,7 +14,7 @@ use serde::Serializer;
 use string_interner::{symbol::SymbolU32, StringInterner, Symbol};
 use syntax_analysis::range::Range;
 
-use crate::{io::read_index_from_file, progress::*};
+use crate::{io::read_index_from_file, progress::*, scip_strict};
 
 pub fn evaluate_command(
     candidate: &Utf8Path,
@@ -731,16 +731,12 @@ impl SymbolFormatter {
 
     fn try_strip_package_details<T: Copy>(&mut self, sym: SymbolId<T>) -> SymbolId<T> {
         let s = self.display_symbol(sym);
-        if s.as_bytes().iter().filter(|&c| *c == b' ').count() != 5 {
+        let Result::Ok(scip_strict::Symbol::NonLocal(mut symbol)) = scip_strict::Symbol::parse(s)
+        else {
             return sym;
-        }
-        let parts: Vec<&str> = s.splitn(5, ' ').collect();
-        let scheme = parts[0];
-        let _manager = parts[1];
-        let _package_name = parts[2];
-        let _version = parts[3];
-        let descriptor = parts[4];
-        self.make_symbol_id(&format!("{scheme} . . . {descriptor}"))
+        };
+        symbol.package = scip_strict::Package::default();
+        self.make_symbol_id(&symbol.to_string())
     }
 }
 

docker-images/syntax-highlighter/crates/scip-syntax/benches/symbol_parsing.rs

@@ -2,3 +2,4 @@ pub mod evaluate;
 pub mod index;
 pub mod io;
 pub mod progress;
+pub mod scip_strict;