Merge pull request #16112 from github/tausbn/python-various-extractor-fixups

Python: Various extractor fixups

Author: RasmusWL

python/extractor/BUILD.bazel

@@ -5,7 +5,6 @@ py_binary(
     srcs = [
         "make_zips.py",
         "python_tracer.py",
-        "unparse.py",
     ],
     data = [
         "LICENSE-PSF.md",

python/extractor/licenses.md

@@ -6,7 +6,6 @@
 | `tsg-python/tree-sitter-python` | Y           | MIT                                      | Used in `tsg-python` to parse Python files                                                                                                                                                     |
 | `tsg-python`                    | Y           | MIT / Apache                             | This is our own creation, so are free to choose what license it is covered by.                                                                                                                 |
 | `tree-sitter-graph`             | N           | MIT / Apache                             | Used in `tsg-python` to execute files written in the `tree-sitter-graph` language.                                                                                                             |
-| `unparse.py`                    | Y           | PSF                                      | Copied and adapted from `Tools/unparse.py` from the `cpython` source code, with attribution.                                                                                                   |
 | `imp.py`                        | Y           | PSF                                      | Copied and adapted from `Lib/imp.py` from the `cpython` source code, with attribution.                                                                                                         |
 | `semmle/data/*.trap`            | Y           | PSF                                      | These files were derived from the C source code of the `cpython` project, and are used in our modelling of built-in objects. No attribution, currently.                                        |
 | `semmle/thrift/parse.py`        | Y           | Apache                                   | Includes a grammar based on https://github.com/apache/thrift/blob/master/doc/specs/idl.md, with comment stating this attribution.                                                              |

python/extractor/make_zips.py

@@ -8,7 +8,6 @@
 import compileall
 
 from python_tracer import getzipfilename
-from unparse import strip_comments_and_docstrings
 
 # TO DO -- Add options to set destination directory and source directory
 
@@ -84,9 +83,7 @@ def write_source(zipped, root, name, extensions=[".py"]):
             if ext not in extensions:
                 continue
             path = os.path.join(dirpath, name)
-            temp = strip_comments_and_docstrings(path)
-            zipped.write(temp, os.path.relpath(path, root))
-            os.remove(temp)
+            zipped.write(path, os.path.relpath(path, root))
 
 def main():
     parser = optparse.OptionParser(usage = "usage: %prog [install-dir]")

python/extractor/tsg-python/Cargo.toml

@@ -7,9 +7,14 @@ authors = ["Taus Brock-Nannestad <[email protected]>"]
 edition = "2018"
 
 # When changing/updating these, the `Cargo.Bazel.lock` file has to be regenerated.
-# Check out the documentation at https://bazelbuild.github.io/rules_rust/crate_universe.html#repinning--updating-dependencies
-# for how to do so. The bazel repository for the tsg-python project is called `py_deps`,
-# and instead of calling `bazel sync`, `./build --bazel sync` should be used instead, to always use the correct bazel version.
+# Run `CARGO_BAZEL_REPIN=true CARGO_BAZEL_REPIN_ONLY=py_deps ./build --bazel sync --only=py_deps`
+# in the `semmle-code` repository to do so.
+# For more information, check out the documentation at
+# https://bazelbuild.github.io/rules_rust/crate_universe.html#repinning--updating-dependencies
+# In the future, the hope is to move this handling of the dependencies entirely into the `codeql` repository,
+# but that depends on `rules_rust` being fully compatible with bzlmod, which they aren't yet
+# (c.f. https://github.com/bazelbuild/rules_rust/issues/2452).
+# Warning: The process takes >5min on my M1 mac, so do wait for a while.
 [dependencies]
 anyhow = "1.0"
 regex = "1"

python/extractor/tsg-python/tsp/BUILD.bazel

@@ -7,7 +7,7 @@ package(default_visibility = ["//visibility:public"])
 # This will run the build script from the root of the workspace, and
 # collect the outputs.
 cargo_build_script(
-    name = "tsg-build",
+    name = "tsp-build",
     srcs = ["bindings/rust/build.rs"],
     data = glob([
         "src/**",
@@ -32,7 +32,7 @@ rust_library(
     proc_macro_deps = all_crate_deps(
         proc_macro = True,
     ),
-    deps = [":tsg-build"] + all_crate_deps(
+    deps = [":tsp-build"] + all_crate_deps(
         normal = True,
     ),
 )