Merge pull request github#16117 from github/redsun82/kotlin

Kotlin: build extractor with bazel

Author: redsun82

.gitattributes

@@ -50,36 +50,37 @@
 *.dll -text
 *.pdb -text
 
-java/ql/test/stubs/**/*.java linguist-generated=true
-java/ql/test/experimental/stubs/**/*.java linguist-generated=true
+/java/ql/test/stubs/**/*.java linguist-generated=true
+/java/ql/test/experimental/stubs/**/*.java linguist-generated=true
+/java/kotlin-extractor/deps/*.jar filter=lfs diff=lfs merge=lfs -text
 
 # Force git not to modify line endings for go or html files under the go/ql directory
-go/ql/**/*.go -text
-go/ql/**/*.html -text
+/go/ql/**/*.go -text
+/go/ql/**/*.html -text
 # Force git not to modify line endings for go dbschemes
-go/*.dbscheme -text
+/go/*.dbscheme -text
 # Preserve unusual line ending from codeql-go merge
-go/extractor/opencsv/CSVReader.java -text
+/go/extractor/opencsv/CSVReader.java -text
 
 # For some languages, upgrade script testing references really old dbscheme
 # files from legacy upgrades that have CRLF line endings. Since upgrade
 # resolution relies on object hashes, we must suppress line ending conversion
 # for those testing dbscheme files.
-*/ql/lib/upgrades/initial/*.dbscheme -text
+/*/ql/lib/upgrades/initial/*.dbscheme -text
 
 # Auto-generated modeling for Python
-python/ql/lib/semmle/python/frameworks/data/internal/subclass-capture/*.yml linguist-generated=true
+/python/ql/lib/semmle/python/frameworks/data/internal/subclass-capture/*.yml linguist-generated=true
 
 # auto-generated bazel lock file
-ruby/extractor/cargo-bazel-lock.json linguist-generated=true
-ruby/extractor/cargo-bazel-lock.json -merge
+/ruby/extractor/cargo-bazel-lock.json linguist-generated=true
+/ruby/extractor/cargo-bazel-lock.json -merge
 
 # auto-generated files for the C# build
-csharp/paket.lock linguist-generated=true
-# needs eol=crlf, as `paket` touches this file and saves it als crlf
-csharp/.paket/Paket.Restore.targets linguist-generated=true eol=crlf
-csharp/paket.main.bzl linguist-generated=true
-csharp/paket.main_extension.bzl linguist-generated=true
+/csharp/paket.lock linguist-generated=true
+# needs eol=crlf, as `paket` touches this file and saves it as crlf
+/csharp/.paket/Paket.Restore.targets linguist-generated=true eol=crlf
+/csharp/paket.main.bzl linguist-generated=true
+/csharp/paket.main_extension.bzl linguist-generated=true
 
 # ripunzip tool
 /misc/bazel/internal/ripunzip/ripunzip-* filter=lfs diff=lfs merge=lfs -text

.github/workflows/kotlin-build.yml

@@ -0,0 +1,28 @@
+name: "Kotlin Build"
+
+on:
+  pull_request:
+    paths:
+      - "java/kotlin-extractor/**"
+      - "misc/bazel/**"
+      - "misc/codegen/**"
+      - "*.bazel*"
+      - .github/workflows/kotlin-build.yml
+    branches:
+      - main
+      - rc/*
+      - codeql-cli-*
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: |
+          bazel query //java/kotlin-extractor/...
+          # only build the default version as a quick check that we can build from `codeql`
+          # the full official build will be checked by QLucie
+          bazel build //java/kotlin-extractor

.lfsconfig

@@ -2,4 +2,6 @@
 # codeql is publicly forked by many users, and we don't want any LFS file polluting their working
 # copies. We therefore exclude everything by default.
 # For files required by bazel builds, use rules in `misc/bazel/lfs.bzl` to download them on demand.
+# we go for `fetchinclude` to something not exsiting rather than `fetchexclude = *` because the
+# former is easier to override (with `git -c` or a local git config) to fetch something specific
 fetchinclude = /nothing

MODULE.bazel

@@ -22,6 +22,7 @@ bazel_dep(name = "bazel_skylib", version = "1.5.0")
 bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "absl")
 bazel_dep(name = "nlohmann_json", version = "3.11.3", repo_name = "json")
 bazel_dep(name = "fmt", version = "10.0.0")
+bazel_dep(name = "rules_kotlin", version = "1.9.4-codeql.1")
 bazel_dep(name = "gazelle", version = "0.36.0")
 bazel_dep(name = "rules_dotnet", version = "0.15.1")
 bazel_dep(name = "googletest", version = "1.14.0.bcr.1")
@@ -65,6 +66,51 @@ node.toolchain(
 )
 use_repo(node, "nodejs", "nodejs_toolchains")
 
+kotlin_extractor_deps = use_extension("//java/kotlin-extractor:deps.bzl", "kotlin_extractor_deps")
+
+# following list can be kept in sync by running `bazel mod tidy` in `codeql`
+use_repo(
+    kotlin_extractor_deps,
+    "codeql_kotlin_defaults",
+    "codeql_kotlin_embeddable",
+    "kotlin-compiler-1.5.0",
+    "kotlin-compiler-1.5.10",
+    "kotlin-compiler-1.5.20",
+    "kotlin-compiler-1.5.30",
+    "kotlin-compiler-1.6.0",
+    "kotlin-compiler-1.6.20",
+    "kotlin-compiler-1.7.0",
+    "kotlin-compiler-1.7.20",
+    "kotlin-compiler-1.8.0",
+    "kotlin-compiler-1.9.0-Beta",
+    "kotlin-compiler-1.9.20-Beta",
+    "kotlin-compiler-2.0.0-RC1",
+    "kotlin-compiler-embeddable-1.5.0",
+    "kotlin-compiler-embeddable-1.5.10",
+    "kotlin-compiler-embeddable-1.5.20",
+    "kotlin-compiler-embeddable-1.5.30",
+    "kotlin-compiler-embeddable-1.6.0",
+    "kotlin-compiler-embeddable-1.6.20",
+    "kotlin-compiler-embeddable-1.7.0",
+    "kotlin-compiler-embeddable-1.7.20",
+    "kotlin-compiler-embeddable-1.8.0",
+    "kotlin-compiler-embeddable-1.9.0-Beta",
+    "kotlin-compiler-embeddable-1.9.20-Beta",
+    "kotlin-compiler-embeddable-2.0.0-RC1",
+    "kotlin-stdlib-1.5.0",
+    "kotlin-stdlib-1.5.10",
+    "kotlin-stdlib-1.5.20",
+    "kotlin-stdlib-1.5.30",
+    "kotlin-stdlib-1.6.0",
+    "kotlin-stdlib-1.6.20",
+    "kotlin-stdlib-1.7.0",
+    "kotlin-stdlib-1.7.20",
+    "kotlin-stdlib-1.8.0",
+    "kotlin-stdlib-1.9.0-Beta",
+    "kotlin-stdlib-1.9.20-Beta",
+    "kotlin-stdlib-2.0.0-RC1",
+)
+
 go_sdk = use_extension("@rules_go//go:extensions.bzl", "go_sdk")
 go_sdk.download(version = "1.22.2")
 

java/downgrades/BUILD.bazel

@@ -0,0 +1,12 @@
+load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix")
+
+pkg_files(
+    name = "downgrades",
+    srcs = glob(
+        ["**"],
+        exclude = ["BUILD.bazel"],
+    ),
+    prefix = "downgrades",
+    strip_prefix = strip_prefix.from_pkg(),
+    visibility = ["//visibility:public"],
+)

java/kotlin-extractor/BUILD.bazel

@@ -0,0 +1,185 @@
+"""
+# Usage overview
+Building the extractor can be done with bazel. If building from the internal repository, it is recommended to use
+`tools/bazel` from there.
+
+A specific kotlin extractor variant can be built with
+```
+bazel build @codeql//java/kotlin-extractor:codeql-extractor-kotlin-<variant>-<version>
+```
+where `<variant>` is either `standalone` or `embeddable`, and `<version>` is one of the supported versions.
+```
+bazel build @codeql//java/kotlin-extractor
+```
+will build a default variant:
+* standalone, unless `CODEQL_KOTLIN_SINGLE_VERSION_EMBEDDABLE` is set to true, in which case it will go for embeddable
+* the version will be taken as the last supported version less than the version of the currently installed `kotlinc`
+    * if `CODEQL_KOTLIN_SINGLE_VERSION` is set, that will be used instead
+    * if `kotlinc` is not installed, `1.9.20-Beta` will be used
+
+If `kotlinc` is updated, bazel won't be aware of it and will therefore keep the same default version. Possible workarounds for that:
+* `bazel clean`
+* `bazel fetch --force @codeql//java/kotlin-extractor`
+* `bazel fetch --force @codeql_kotlin_defaults//:all` (only from `codeql`)
+
+If building from the `codeql` repository, `@codeql` can be skipped.
+"""
+
+# This file is used in the `@codeql_kotlin_embeddable` external repo, which means we need to
+# reference explicitly @codeql
+load(
+    "@codeql//java/kotlin-extractor:versions.bzl",
+    "VERSIONS",
+    "get_compatilibity_sources",
+    "get_language_version",
+    "version_less",
+)
+load("@rules_kotlin//kotlin:core.bzl", "kt_javac_options", "kt_kotlinc_options")
+load("@rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library")
+
+package(default_visibility = ["//java/kotlin-extractor:__subpackages__"])
+
+_for_embeddable = repo_name().endswith("codeql_kotlin_embeddable")
+
+_common_extractor_name_prefix = "codeql-extractor-kotlin"
+
+_extractor_name_prefix = "%s-%s" % (
+    _common_extractor_name_prefix,
+    "embeddable" if _for_embeddable else "standalone",
+)
+
+py_binary(
+    name = "generate_dbscheme",
+    srcs = ["generate_dbscheme.py"],
+)
+
+_resources = [
+    (
+        r,
+        r[len("src/main/resources/"):],
+    )
+    for r in glob(["src/main/resources/**"])
+]
+
+kt_javac_options(
+    name = "javac-options",
+    release = "8",
+)
+
+[
+    (
+        kt_kotlinc_options(
+            name = "kotlinc-options-%s" % v,
+            include_stdlibs = "none",
+            jvm_target = "1.8",
+            language_version = get_language_version(v),
+            warn = "error",
+            x_optin = [
+                "kotlin.RequiresOptIn",
+                "org.jetbrains.kotlin.ir.symbols.%s" %
+                ("IrSymbolInternals" if version_less(v, "2.0.0") else "UnsafeDuringIrConstructionAPI"),
+            ],
+            x_suppress_version_warnings = True,
+        ),
+        # * extractor.name is different for each version, so we need to put it in different output dirs
+        # * in order to put it in `resources`, we need to define `resource_strip_prefix` to strip this version
+        # * `resource_strip_prefix` is unique per jar, so we must also put other resources under the same version prefix
+        genrule(
+            name = "resources-%s" % v,
+            srcs = [src for src, _ in _resources],
+            outs = [
+                "%s/com/github/codeql/extractor.name" % v,
+            ] + [
+                "%s/%s" % (v, target)
+                for _, target in _resources
+            ],
+            cmd = "\n".join([
+                "echo %s-%s > $(RULEDIR)/%s/com/github/codeql/extractor.name" % (_extractor_name_prefix, v, v),
+            ] + [
+                "cp $(execpath %s) $(RULEDIR)/%s/%s" % (source, v, target)
+                for source, target in _resources
+            ]),
+        ),
+        kt_jvm_library(
+            name = "%s-%s" % (_extractor_name_prefix, v),
+            srcs =
+                ["@codeql//java/kotlin-extractor:generated-dbscheme"] +
+                glob(
+                    [
+                        "src/**/*.kt",
+                        "src/**/*.java",
+                    ],
+                    exclude = ["src/main/kotlin/utils/versions/**"],
+                ) + get_compatilibity_sources(v, "src/main/kotlin/utils/versions"),
+            javac_opts = ":javac-options",
+            kotlinc_opts = ":kotlinc-options-%s" % v,
+            module_name = "codeql-kotlin-extractor",
+            # resource_strip_prefix is very nit-picky: the following makes it work from
+            # `codeql`, `@codeql_kotlin_embeddable` and `semmle-code`
+            resource_strip_prefix = (
+                ("../%s/" % repo_name() if repo_name() else "") +
+                ("%s/" % package_name() if package_name() else "") +
+                v
+            ),
+            resources = [
+                ":resources-%s" % v,
+            ],
+            visibility = ["//visibility:public"],
+            deps = [
+                "@kotlin-compiler%s-%s" % (
+                    "-embeddable" if _for_embeddable else "",
+                    v,
+                ),
+                "@kotlin-stdlib-%s" % v,
+            ],
+        ),
+        # if in main repository, alias the embeddable versions from the modified @codeql_kotlin_embeddable repo
+        alias(
+            name = "%s-embeddable-%s" % (_common_extractor_name_prefix, v),
+            actual = "@codeql_kotlin_embeddable//:%s-embeddable-%s" % (_common_extractor_name_prefix, v),
+            visibility = ["//visibility:public"],
+        ) if not _for_embeddable else None,
+    )
+    for v in VERSIONS
+]
+
+(
+    genrule(
+        name = "generated-dbscheme",
+        srcs = ["@codeql//java:dbscheme"],
+        outs = ["KotlinExtractorDbScheme.kt"],
+        cmd = "$(execpath :generate_dbscheme) $< $@",
+        tools = [":generate_dbscheme"],
+        visibility = ["@codeql_kotlin_embeddable//:__pkg__"],
+    ),
+    [
+        alias(
+            name = n,
+            actual = "//java/kotlin-extractor/defaults:%s" % n,
+            visibility = ["//visibility:public"],
+        )
+        for n in (
+            "%s-standalone" % _common_extractor_name_prefix,
+            "%s-embeddable" % _common_extractor_name_prefix,
+            _common_extractor_name_prefix,
+        )
+    ],
+    alias(
+        name = "kotlin-extractor",
+        actual = _common_extractor_name_prefix,
+        visibility = ["//visibility:public"],
+    ),
+    filegroup(
+        name = "many",
+        srcs = ["%s-%s-%s" % (
+            _common_extractor_name_prefix,
+            variant,
+            version,
+        ) for variant in ("standalone", "embeddable") for version in VERSIONS],
+        visibility = ["//visibility:public"],
+    ),
+    sh_binary(
+        name = "print-default-version",
+        srcs = ["//java/kotlin-extractor/defaults:default-version-printer"],
+    ),
+) if not _for_embeddable else None

java/kotlin-extractor/current_kotlin_version.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python3
+
+import subprocess
+import re
+import shutil
+
+kotlinc = shutil.which('kotlinc')
+if kotlinc is None:
+    raise Exception("kotlinc not found")
+res = subprocess.run([kotlinc, "-version"], text=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
+if res.returncode != 0:
+    raise Exception(f"kotlinc -version failed: {res.stderr}")
+m = re.match(r'.* kotlinc-jvm ([0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z][a-zA-Z0-9]*)?) .*', res.stderr)
+if m is None:
+    raise Exception(f'Cannot detect version of kotlinc (got {res.stderr})')
+print(m[1])