feat: upgrade language support

Author: williamfzc

Dockerfile

@@ -4,11 +4,6 @@ RUN apk add --no-cache graphviz curl git
 
 COPY . /action_internal
 
-# python lsif
-RUN apk add --no-cache python3 py3-pip && \
-    pip3 install -r /action_internal/requirements.txt && \
-    pip3 install --upgrade git+https://github.com/sourcegraph/lsif-py.git
-
 # scip converter
 RUN git clone https://github.com/sourcegraph/scip.git --depth=1 ./scip_repo && \
     cd scip_repo && \
@@ -26,6 +21,15 @@ RUN apk add openjdk8 gradle maven \
   && ./coursier bootstrap --standalone -o scip-java com.sourcegraph:scip-java_2.13:0.8.18 --main com.sourcegraph.scip_java.ScipJava \
   && ./scip-java --help
 
+# node lsif and python scip
+RUN apk add --update nodejs npm \
+  && npm install -g lsif \
+  && npm install -g @sourcegraph/scip-python
+
+# python runtime for main script
+RUN apk add --no-cache python3 py3-pip && \
+    pip3 install -r /action_internal/requirements.txt
+
 ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk
 
 ENTRYPOINT ["python3", "/action_internal/main.py"]

debug.py

@@ -1,20 +1,47 @@
 import os
+import pathlib
 
+import config
 from diff import gen_diff
 from index import gen_index
 from utils import check_call
 import shutil
 
 
 def debug_main():
-    # pull some repos for test
-    check_call(["git", "clone", "--depth=2", "https://github.com/gin-gonic/gin.git"])
-
-    gen_index("golang", "gin", "")
+    # clone junit4 repository
+    check_call(
+        ["git", "clone", "--depth=2", "https://github.com/junit-team/junit4.git"]
+    )
+    junit4_dir = pathlib.Path(config.USER_DIR) / "junit4"
+    gen_index(
+        "java",
+        junit4_dir.as_posix(),
+        "scip-java index -- "
+        "package -DskipTests "
+        "--batch-mode "
+        "--errors "
+        "--settings .github/workflows/settings.xml",
+    )
+    os.chdir(junit4_dir)
+    gen_diff(junit4_dir.as_posix(), "HEAD~1", "HEAD", "")
+    os.chdir(config.USER_DIR)
+    shutil.rmtree(junit4_dir)
 
-    os.chdir("gin")
-    gen_diff("HEAD~1", "HEAD", "")
-    os.chdir("..")
+    # clone gin repository
+    check_call(["git", "clone", "--depth=2", "https://github.com/gin-gonic/gin.git"])
+    gin_dir = pathlib.Path(config.USER_DIR) / "gin"
+    os.chdir(gin_dir)
+    gen_index("golang", gin_dir.as_posix(), "")
+    gen_diff(gin_dir.as_posix(), "HEAD~1", "HEAD", "")
+    os.chdir(config.USER_DIR)
+    shutil.rmtree(gin_dir)
 
-    # clean up
-    shutil.rmtree("gin")
+    # clone requests repository
+    check_call(["git", "clone", "--depth=2", "https://github.com/psf/requests.git"])
+    requests_dir = pathlib.Path(config.USER_DIR) / "requests"
+    os.chdir(requests_dir)
+    gen_index("python", requests_dir.as_posix(), "")
+    gen_diff(requests_dir.as_posix(), "HEAD~1", "HEAD", "")
+    os.chdir(config.USER_DIR)
+    shutil.rmtree(requests_dir)

diff.py

@@ -1,16 +1,17 @@
+import pathlib
+
 import config
 from utils import check_call
 
 
-def gen_diff(before_sha: str, after_sha: str, lsif_file: str):
-    # gen diff
-    set_safe_git_dir()
+def gen_diff(src_dir: str, before_sha: str, after_sha: str, lsif_file: str):
+    set_safe_git_dir(src_dir)
 
     cmds = [
         "srctx",
         "diff",
         "--src",
-        config.USER_DIR,
+        src_dir,
         "--before",
         before_sha,
         "--after",
@@ -20,12 +21,23 @@ def gen_diff(before_sha: str, after_sha: str, lsif_file: str):
         "--outputCsv",
         config.CSV_RESULT_FILE,
     ]
-    if lsif_file:
+    if not lsif_file:
+        lsif_file_loc = pathlib.Path(src_dir) / "dump.lsif"
+        scip_file_loc = pathlib.Path(src_dir) / "index.scip"
+
+        if lsif_file_loc.is_file():
+            cmds += ["--lsif", str(lsif_file_loc.absolute())]
+        elif scip_file_loc.is_file():
+            cmds += ["--scip", str(scip_file_loc.absolute())]
+        else:
+            raise RuntimeError(f"no index file found in {src_dir}")
+    else:
         cmds += ["--lsif", lsif_file]
+
     check_call(cmds)
 
 
-def set_safe_git_dir():
+def set_safe_git_dir(src_dir: str):
     check_call(
-        ["git", "config", "--global", "--add", "safe.directory", config.USER_DIR]
+        ["git", "config", "--global", "--add", "safe.directory", src_dir]
     )

index.py

@@ -13,6 +13,7 @@ def gen_index(lang: str, directory: str, index_command: str):
         if index_command:
             logger.info(f"custom index command: {index_command}")
             check_call(index_command.split(" "))
+            return
 
         if lang == "golang":
             gen_golang_index()
@@ -22,9 +23,10 @@ def gen_index(lang: str, directory: str, index_command: str):
             gen_java_and_kotlin_index()
         elif lang == "kotlin":
             gen_java_and_kotlin_index()
+        elif lang == "node":
+            gen_node_index()
         else:
-            logger.error("no index mapping")
-            return
+            raise RuntimeError(f"lang {lang} not support")
     finally:
         os.chdir(current_directory)
 
@@ -36,9 +38,21 @@ def gen_golang_index():
 def gen_java_and_kotlin_index():
     # https://sourcegraph.github.io/scip-java/docs/getting-started.html#run-scip-java-index
     check_call(["scip-java", "index", "--output", "index.scip"])
-    # https://github.com/sourcegraph/scip/blob/main/docs/CLI.md
-    check_call(["scip", "convert", "--from", "index.scip", "--to", "dump.lsif"])
 
 
 def gen_py_index():
-    check_call(["lsif-py", ".", "--file", "./dump.lsif"])
+    check_call(["scip-python", "index", ".", "--project-name", "srctx"])
+
+
+def gen_node_index():
+    check_call(
+        [
+            "lsif",
+            "tsc",
+            "-p",
+            "./tsconfig.json",
+            "--package",
+            "./package.json",
+            "--stdout",
+        ]
+    )

main.py

@@ -48,13 +48,13 @@ def main():
         config.USER_DIR = "."
 
     # data prepare
-    set_safe_git_dir()
+    set_safe_git_dir(config.USER_DIR)
     files = os.listdir(config.USER_DIR)
     logger.info(f"files: {files}")
 
     if not lsif_file:
         gen_index(lang, config.USER_DIR, index_command)
-    gen_diff(before_sha, after_sha, lsif_file)
+    gen_diff(config.USER_DIR, before_sha, after_sha, lsif_file)
 
     repo_name = os.getenv("GITHUB_REPOSITORY")
     file_list = load_index_data()
@@ -160,7 +160,9 @@ def dot_to_svg(dot_file):
     return svg_bytes
 
 
-def sort_files_by_impact(files: typing.Iterable[FileMetrics]) -> typing.List[FileMetrics]:
+def sort_files_by_impact(
+        files: typing.Iterable[FileMetrics],
+) -> typing.List[FileMetrics]:
     def sort_key(f: FileMetrics):
         return f.affectedLineCount
 

test_index.py

@@ -1,4 +1,3 @@
-from index import gen_index
 from main import export_csv_table, load_index_data
 
 

utils.py

@@ -1,5 +1,7 @@
 import subprocess
+from loguru import logger
 
 
 def check_call(commands: list):
+    logger.info(f"check calling: {commands}")
     subprocess.check_call(commands)