Merge branch 'main' into rust-df-patterns

Author: paldepind

.bazelrc

@@ -23,6 +23,5 @@ common --registry=file:///%workspace%/misc/bazel/registry
 common --registry=https://bcr.bazel.build
 
 common --@rules_dotnet//dotnet/settings:strict_deps=false
-common --experimental_isolated_extension_usages
 
 try-import %workspace%/local.bazelrc

.bazelrc.internal

@@ -8,4 +8,3 @@ common --registry=https://bcr.bazel.build
 # its implementation packages without providing any code itself.
 # We either can depend on internal implementation details, or turn of strict deps.
 common --@rules_dotnet//dotnet/settings:strict_deps=false
-common --experimental_isolated_extension_usages

BUILD.bazel

@@ -1 +1,5 @@
-exports_files(["LICENSE"])
+exports_files([
+    "LICENSE",
+    "Cargo.lock",
+    "Cargo.toml",
+])

MODULE.bazel

@@ -31,41 +31,44 @@ bazel_dep(name = "rules_rust", version = "0.52.2")
 
 bazel_dep(name = "buildifier_prebuilt", version = "6.4.0", dev_dependency = True)
 
-# crate_py but shortened due to Windows file path considerations
-cp = use_extension(
-    "@rules_rust//crate_universe:extension.bzl",
-    "crate",
-    isolate = True,
-)
-cp.from_cargo(
-    name = "py_deps",
-    cargo_lockfile = "//python/extractor/tsg-python:Cargo.lock",
-    manifests = [
-        "//python/extractor/tsg-python:Cargo.toml",
-        "//python/extractor/tsg-python/tsp:Cargo.toml",
+# Keep edition and version approximately in sync with internal repo.
+# the versions there are canonical, the versions here are used for CI in github/codeql, as well as for the vendoring of dependencies.
+RUST_EDITION = "2021"
+
+RUST_VERSION = "1.81.0"
+
+rust = use_extension("@rules_rust//rust:extensions.bzl", "rust")
+rust.toolchain(
+    edition = RUST_EDITION,
+    # We need those extra target triples so that we can build universal binaries on macos
+    extra_target_triples = [
+        "x86_64-apple-darwin",
+        "aarch64-apple-darwin",
     ],
+    versions = [RUST_VERSION],
 )
-use_repo(cp, "py_deps")
+use_repo(rust, "rust_toolchains")
 
-# deps for ruby+rust, but shortened due to windows file paths
-r = use_extension(
-    "@rules_rust//crate_universe:extension.bzl",
-    "crate",
-    isolate = True,
-)
-r.from_cargo(
-    name = "r",
-    cargo_lockfile = "//:Cargo.lock",
-    manifests = [
-        "//:Cargo.toml",
-        "//ruby/extractor:Cargo.toml",
-        "//rust/extractor:Cargo.toml",
-        "//rust/extractor/macros:Cargo.toml",
-        "//rust/ast-generator:Cargo.toml",
-        "//shared/tree-sitter-extractor:Cargo.toml",
-    ],
+register_toolchains("@rust_toolchains//:all")
+
+rust_host_tools = use_extension("@rules_rust//rust:extensions.bzl", "rust_host_tools")
+
+# Don't download a second toolchain as host toolchain, make sure this is the same version as above
+# The host toolchain is used for vendoring dependencies.
+rust_host_tools.host_tools(
+    edition = RUST_EDITION,
+    version = RUST_VERSION,
 )
-use_repo(r, tree_sitter_extractors_deps = "r")
+
+# deps for python extractor
+# keep in sync by running `misc/bazel/3rdparty/update_cargo_deps.sh`
+py_deps = use_extension("//misc/bazel/3rdparty:py_deps_extension.bzl", "p")
+use_repo(py_deps, "vendor__anyhow-1.0.44", "vendor__cc-1.0.70", "vendor__clap-2.33.3", "vendor__regex-1.5.5", "vendor__smallvec-1.6.1", "vendor__string-interner-0.12.2", "vendor__thiserror-1.0.29", "vendor__tree-sitter-0.20.4", "vendor__tree-sitter-graph-0.7.0")
+
+# deps for ruby+rust
+# keep in sync by running `misc/bazel/3rdparty/update_cargo_deps.sh`
+tree_sitter_extractors_deps = use_extension("//misc/bazel/3rdparty:tree_sitter_extractors_extension.bzl", "r")
+use_repo(tree_sitter_extractors_deps, "vendor__anyhow-1.0.93", "vendor__argfile-0.2.1", "vendor__chrono-0.4.38", "vendor__clap-4.5.20", "vendor__encoding-0.2.33", "vendor__figment-0.10.19", "vendor__flate2-1.0.34", "vendor__glob-0.3.1", "vendor__globset-0.4.15", "vendor__itertools-0.10.5", "vendor__itertools-0.13.0", "vendor__lazy_static-1.5.0", "vendor__log-0.4.22", "vendor__num-traits-0.2.19", "vendor__num_cpus-1.16.0", "vendor__proc-macro2-1.0.89", "vendor__quote-1.0.37", "vendor__ra_ap_base_db-0.0.232", "vendor__ra_ap_hir-0.0.232", "vendor__ra_ap_hir_def-0.0.232", "vendor__ra_ap_hir_expand-0.0.232", "vendor__ra_ap_ide_db-0.0.232", "vendor__ra_ap_load-cargo-0.0.232", "vendor__ra_ap_parser-0.0.232", "vendor__ra_ap_paths-0.0.232", "vendor__ra_ap_project_model-0.0.232", "vendor__ra_ap_span-0.0.232", "vendor__ra_ap_syntax-0.0.232", "vendor__ra_ap_vfs-0.0.232", "vendor__rand-0.8.5", "vendor__rayon-1.10.0", "vendor__regex-1.11.1", "vendor__serde-1.0.214", "vendor__serde_json-1.0.132", "vendor__serde_with-3.11.0", "vendor__stderrlog-0.6.0", "vendor__syn-2.0.87", "vendor__tracing-0.1.40", "vendor__tracing-subscriber-0.3.18", "vendor__tree-sitter-0.24.4", "vendor__tree-sitter-embedded-template-0.23.2", "vendor__tree-sitter-json-0.24.8", "vendor__tree-sitter-ql-0.23.1", "vendor__tree-sitter-ruby-0.23.1", "vendor__triomphe-0.1.14", "vendor__ungrammar-1.16.1")
 
 dotnet = use_extension("@rules_dotnet//dotnet:extensions.bzl", "dotnet")
 dotnet.toolchain(dotnet_version = "8.0.101")

csharp/CSharp.sln

@@ -4,8 +4,6 @@ VisualStudioVersion = 15.0.27130.2036
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Util", "extractor\Semmle.Util\Semmle.Util.csproj", "{CDD7AD69-0FD8-40F0-A9DA-F1077A2A85D6}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction", "extractor\Semmle.Extraction\Semmle.Extraction.csproj", "{81EAAD75-4BE1-44E4-91DF-20778216DB64}"
-EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction.CSharp", "extractor\Semmle.Extraction.CSharp\Semmle.Extraction.CSharp.csproj", "{C4D62DA0-B64B-440B-86DC-AB52318CB8BF}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction.CSharp.DependencyFetching", "extractor\Semmle.Extraction.CSharp.DependencyFetching\Semmle.Extraction.CSharp.DependencyFetching.csproj", "{541D1AC5-E42C-4AB2-A1A4-C2355CE2A2EF}"

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/BUILD.bazel

@@ -14,7 +14,7 @@ codeql_csharp_library(
     nowarn = ["CA1822"],
     visibility = ["//csharp:__subpackages__"],
     deps = [
-        "//csharp/extractor/Semmle.Extraction",
+        "//csharp/extractor/Semmle.Extraction.CSharp",
         "//csharp/extractor/Semmle.Util",
     ],
 )

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetExeWrapper.cs

@@ -15,7 +15,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
     internal class NugetExeWrapper : IDisposable
     {
         private readonly string? nugetExe;
-        private readonly Util.Logging.ILogger logger;
+        private readonly Semmle.Util.Logging.ILogger logger;
 
         public int PackageCount => fileProvider.PackagesConfigs.Count;
 
@@ -33,7 +33,7 @@ internal class NugetExeWrapper : IDisposable
         /// <summary>
         /// Create the package manager for a specified source tree.
         /// </summary>
-        public NugetExeWrapper(FileProvider fileProvider, TemporaryDirectory packageDirectory, Util.Logging.ILogger logger)
+        public NugetExeWrapper(FileProvider fileProvider, TemporaryDirectory packageDirectory, Semmle.Util.Logging.ILogger logger)
         {
             this.fileProvider = fileProvider;
             this.packageDirectory = packageDirectory;

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Semmle.Extraction.CSharp.DependencyFetching.csproj

@@ -5,7 +5,7 @@
   </PropertyGroup>
   <ItemGroup>
     <ProjectReference Include="..\Semmle.Util\Semmle.Util.csproj" />
-    <ProjectReference Include="..\Semmle.Extraction\Semmle.Extraction.csproj" />
+    <ProjectReference Include="..\Semmle.Extraction.CSharp\Semmle.Extraction.CSharp.csproj" />
 
     <InternalsVisibleTo Include="Semmle.Extraction.Tests" />
   </ItemGroup>

csharp/extractor/Semmle.Extraction.CSharp/BUILD.bazel

@@ -6,17 +6,18 @@ load(
 codeql_csharp_library(
     name = "Semmle.Extraction.CSharp",
     srcs = glob([
+        "CodeAnalysisExtensions/**/*.cs",
         "Comments/**/*.cs",
         "Entities/**/*.cs",
         "Extractor/**/*.cs",
         "Kinds/**/*.cs",
         "Populators/**/*.cs",
+        "Trap/**/*.cs",
         "*.cs",
     ]),
     allow_unsafe_blocks = True,
     visibility = ["//csharp:__subpackages__"],
     deps = [
-        "//csharp/extractor/Semmle.Extraction",
         "//csharp/extractor/Semmle.Extraction.CSharp.Util",
         "//csharp/extractor/Semmle.Util",
         "@paket.main//basic.compilerlog.util",