Fix workspace resolution for development (#1231)

Align versions tween bzlmod and workspace
Switch to the fancy bazel ci rules for rbe.
Enable RBE for integration tests (huuuuge difference)

Author: restingbull

.bazelci/presubmit.yml

@@ -10,9 +10,9 @@ matrix:
     # - windows re-enable when rules_bazel_integration_test can support custom test runner on windows.
   test_flags:
     - ["--enable_bzlmod=true"]
-    - ["--enable_bzlmod=false"]
+    - ["--config=workspace"]
 validate_config: 1
-bazel: 7.1.1
+bazel: 7.3.0
 buildifier:
   version: 7.1.1
   # no lint warnings for the moment. They are basically a smoke alarm in hell.
@@ -25,22 +25,17 @@ tasks:
     build_flags: ${{ test_flags }}
     test_flags: ${{ test_flags }}
     test_targets:
-      - "//:all_tests"
+      - "//src/..."
     build_targets:
-      - "//:rules_kotlin_release"
+      - "//src/..."
   integration_tests:
     name: "Integration Tests"
     platform: ${{ integration_platform }}
     test_flags:
-      - "--enable_bzlmod=true"
+      - "--config=rbe"
     test_targets:
       - //examples:all
   rbe_ubuntu1604:
-    build_flags:
-      - "--enable_bzlmod=false"
-      - "--config=rbe"
-      - "--host_java_toolchain=@bazel_tools//tools/jdk:toolchain_java11"
-      - "--java_toolchain=@bazel_tools//tools/jdk:toolchain_java11"
     test_targets:
       - "--"
       - "//src/test/kotlin/io/bazel/kotlin/builder:builder_tests"
@@ -51,8 +46,8 @@ tasks:
     test_flags:
       # Override the default worker strategy for remote builds (worker strategy
       # cannot be used with remote builds)
-      - "--strategy=KotlinCompile=remote"
       - "--config=rbe"
+      - "--strategy=KotlinCompile=remote"
   stardoc:
     name: Stardoc api documentation
     platform: ubuntu1804

.bazelrc

@@ -3,6 +3,8 @@ common --enable_bzlmod=true
 common:rbe --java_runtime_version=11
 common:rbe --tool_java_runtime_version=11
 
+common:workspace --enable_bzlmod=false --enable_workspace=true
+
 build --strategy=KotlinCompile=worker
 build --test_output=all
 build --verbose_failures

MODULE.bazel

@@ -7,15 +7,14 @@ module(
 
 bazel_dep(name = "platforms", version = "0.0.10")
 bazel_dep(name = "bazel_skylib", version = "1.7.1")
-bazel_dep(name = "rules_java", version = "7.2.0")
+bazel_dep(name = "rules_java", version = "7.6.5")
 bazel_dep(name = "rules_python", version = "0.23.1")
 bazel_dep(name = "rules_cc", version = "0.0.9")
 bazel_dep(name = "rules_android", version = "0.1.1")
 
 rules_kotlin_extensions = use_extension("//src/main/starlark/core/repositories:bzlmod_setup.bzl", "rules_kotlin_extensions")
 use_repo(
     rules_kotlin_extensions,
-    "buildkite_config",
     "com_github_google_ksp",
     "com_github_jetbrains_kotlin",
     "com_github_jetbrains_kotlin_git",
@@ -51,6 +50,14 @@ use_repo(remote_android_extensions, "android_gmaven_r8", "android_tools")
 # we need to sort out a few cases around how these rules are consumed in various ways.
 
 bazel_dep(name = "rules_jvm_external", version = "5.3")
+bazel_dep(name = "bazel_ci_rules", version = "1.0.0")
+
+rbe_preconfig = use_repo_rule("@bazel_ci_rules//:rbe_repo.bzl", "rbe_preconfig")
+
+rbe_preconfig(
+    name = "buildkite_config",
+    toolchain = "ubuntu2004",
+)
 
 maven = use_extension("@rules_jvm_external//:extensions.bzl", "maven")
 maven.install(

src/main/starlark/core/repositories/BUILD

@@ -39,7 +39,4 @@ bzl_library(
     name = "repositories",
     srcs = glob(["*.bzl"]),
     visibility = ["//:__subpackages__"],
-    deps = [
-        "@rules_proto//proto:defs",
-    ],
 )

src/main/starlark/core/repositories/bzlmod_setup.bzl

@@ -32,13 +32,6 @@ def _rules_kotlin_extensions_impl(mctx):
         ],
     )
 
-    # This tarball intentionally does not have a SHA256 because the upstream URL can change without notice
-    # For more context: https://github.com/bazelbuild/bazel-toolchains/blob/0c1f7c3c5f9e63f1e0ee91738b964937eea2d3e0/WORKSPACE#L28-L32
-    http_archive(
-        name = "buildkite_config",
-        urls = _versions.RBE.URLS,
-    )
-
     return modules.use_all_repos(mctx, reproducible = True)
 
 rules_kotlin_extensions = module_extension(

src/main/starlark/core/repositories/download.bzl

@@ -20,10 +20,25 @@ def kt_download_local_dev_dependencies():
     """
     Downloads all necessary http_* artifacts for rules_kotlin dev configuration.
 
-    Must be called before setup_dependencies in the versions.WORKSPACE.
+    Must be called before setup_dependencies in the WORKSPACE.
     """
+    versions.use_repository(
+        name = "rules_proto",
+        version = versions.RULES_PROTO,
+        rule = rules_stardoc_repository,
+        starlark_packages = [
+            "proto",
+            "proto/private",
+        ],
+    )
 
-    # bazel_skylib is initialized twice during developement. This is intentional, as development
+    versions.use_repository(
+        rule = http_archive,
+        name = "rules_cc",
+        version = versions.RULES_CC,
+    )
+
+    # bazel_skylib is initialized twice during development. This is intentional, as development
     # needs to be able to run the starlark unittests, while production does not.
     maybe(
         http_archive,
@@ -43,12 +58,16 @@ def kt_download_local_dev_dependencies():
         ],
     )
 
-    # This tarball intentionally does not have a SHA256 because the upstream URL can change without notice
-    # For more context: https://github.com/bazelbuild/bazel-toolchains/blob/0c1f7c3c5f9e63f1e0ee91738b964937eea2d3e0/WORKSPACE#L28-L32
-    maybe(
-        http_archive,
-        name = "buildkite_config",
-        urls = versions.RBE.URLS,
+    versions.use_repository(
+        name = "rules_python",
+        rule = http_archive,
+        version = versions.RULES_PYTHON,
+    )
+
+    versions.use_repository(
+        name = "rules_java",
+        rule = http_archive,
+        version = versions.RULES_JAVA,
     )
 
     maybe(
@@ -88,13 +107,22 @@ def kt_download_local_dev_dependencies():
     )
 
     versions.use_repository(
-        name = "rules_proto",
-        version = versions.RULES_PROTO,
-        rule = rules_stardoc_repository,
-        starlark_packages = [
-            "proto",
-            "proto/private",
-        ],
+        name = "rules_testing",
+        rule = http_archive,
+        version = versions.RULES_TESTING,
+        strip_prefix = "rules_testing-%s" % versions.RULES_TESTING.version,
+    )
+
+    versions.use_repository(
+        name = "rules_bazel_integration_test",
+        rule = http_archive,
+        version = versions.RULES_BAZEL_INTEGRATION_TEST,
+    )
+
+    versions.use_repository(
+        name = "cgrindel_bazel_starlib",
+        rule = http_archive,
+        version = versions.CGRINDEL_BAZEL_STARLIB,
     )
 
     versions.use_repository(

src/main/starlark/core/repositories/initialize.release.bzl

@@ -104,12 +104,6 @@ def kotlin_repositories(
         urls = versions.ANDROID.URLS,
     )
 
-    versions.use_repository(
-        name = "rules_python",
-        rule = http_archive,
-        version = versions.RULES_PYTHON,
-    )
-
     versions.use_repository(
         name = "rules_java",
         rule = http_archive,

src/main/starlark/core/repositories/setup.bzl

@@ -13,8 +13,12 @@
 # limitations under the License.
 
 load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace")
+load("@cgrindel_bazel_starlib//:deps.bzl", "bazel_starlib_dependencies")
 load("@io_bazel_stardoc//:setup.bzl", "stardoc_repositories")
 load("@released_rules_kotlin//src/main/starlark/core/repositories:initialize.bzl", release_kotlin_repositories = "kotlin_repositories")
+load("@rules_bazel_integration_test//bazel_integration_test:deps.bzl", "bazel_integration_test_rules_dependencies")
+load("@rules_cc//cc:repositories.bzl", "rules_cc_dependencies", "rules_cc_toolchains")
+load("@rules_java//java:repositories.bzl", "rules_java_dependencies", "rules_java_toolchains")
 load("@rules_jvm_external//:defs.bzl", "maven_install")
 load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies")
 load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains")
@@ -29,6 +33,15 @@ def kt_configure():
 
     native.register_toolchains("@released_rules_kotlin//kotlin/internal:default_toolchain")
 
+    rules_cc_dependencies()
+    rules_cc_toolchains()
+
+    rules_proto_dependencies()
+    rules_proto_toolchains()
+
+    rules_java_dependencies()
+    rules_java_toolchains()
+
     maven_install(
         name = "kotlin_rules_maven",
         fetch_sources = True,
@@ -39,10 +52,12 @@ def kt_configure():
             "com.google.protobuf:protobuf-java-util:3.6.0",
             "com.google.guava:guava:27.1-jre",
             "com.google.truth:truth:0.45",
-            "com.google.auto.service:auto-service:1.0.1",
-            "com.google.auto.service:auto-service-annotations:1.0.1",
+            "com.google.auto.service:auto-service:1.1.1",
+            "com.google.auto.service:auto-service-annotations:1.1.1",
+            "com.google.auto.service:auto-service-annotations:jar:1.1.1",
             "com.google.auto.value:auto-value:1.10.1",
             "com.google.auto.value:auto-value-annotations:1.10.1",
+            "org.apache.commons:commons-compress:1.26.2",
             "com.google.dagger:dagger:2.51",
             "com.google.dagger:dagger-compiler:2.51",
             "com.google.dagger:dagger-producers:2.51",
@@ -55,7 +70,6 @@ def kt_configure():
             "com.squareup.moshi:moshi:1.15.0",
             "com.squareup.moshi:moshi-kotlin:1.15.0",
             "com.squareup.moshi:moshi-kotlin-codegen:1.15.0",
-            "com.google.auto.service:auto-service-annotations:jar:1.1.1",
         ],
         repositories = [
             "https://maven-central.storage.googleapis.com/repos/central/data/",
@@ -64,11 +78,15 @@ def kt_configure():
         ],
     )
 
-    rules_proto_dependencies()
-    rules_proto_toolchains()
+    rules_cc_dependencies()
+    rules_cc_toolchains()
 
     rules_pkg_dependencies()
 
     stardoc_repositories()
 
     bazel_skylib_workspace()
+
+    bazel_integration_test_rules_dependencies()
+    bazel_starlib_dependencies()
+    bazel_skylib_workspace()

src/main/starlark/core/repositories/versions.bzl

@@ -20,8 +20,6 @@ def _use_repository(name, version, rule, **kwargs):
     maybe(rule, name = name, **http_archive_arguments)
 
 versions = struct(
-    RULES_NODEJS_VERSION = "5.5.3",
-    RULES_NODEJS_SHA = "f10a3a12894fc3c9bf578ee5a5691769f6805c4be84359681a785a0c12e8d2b6",
     BAZEL_TOOLCHAINS_VERSION = "4.1.0",
     BAZEL_TOOLCHAINS_SHA = "179ec02f809e86abf56356d8898c8bd74069f1bd7c56044050c2cd3d79d0e024",
     # IMPORTANT! rules_kotlin does not use the bazel_skylib unittest in production
@@ -33,8 +31,6 @@ versions = struct(
     # --> 3. Configure dependencies <--
     SKYLIB_VERSION = "1.4.2",
     SKYLIB_SHA = "66ffd9315665bfaafc96b52278f57c7e2dd09f5ede279ea6d39b2be471e7e3aa",
-    PROTOBUF_VERSION = "3.11.3",
-    PROTOBUF_SHA = "cf754718b0aa945b00550ed7962ddc167167bd922b842199eeb6505e6f344852",
     RULES_JVM_EXTERNAL_TAG = "5.3",
     RULES_JVM_EXTERNAL_SHA = "d31e369b854322ca5098ea12c69d7175ded971435e55c18dd9dd5f29cc5249ac",
     RULES_PROTO = version(
@@ -80,17 +76,19 @@ versions = struct(
         URLS = ["https://github.com/bazelbuild/rules_android/archive/v%s.zip" % "0.1.1"],
     ),
     # To update: https://github.com/bazelbuild/bazel-toolchains#latest-bazel-and-latest-ubuntu-1604-container
-    RBE = struct(
+    BAZELCI_RULES = struct(
+        version = "1.0.0",
+        sha256 = "eca21884e6f66a88c358e580fd67a6b148d30ab57b1680f62a96c00f9bc6a07e",
         # This tarball intentionally does not have a SHA256 because the upstream URL can change without notice
         # For more context: https://github.com/bazelbuild/bazel-toolchains/blob/0c1f7c3c5f9e63f1e0ee91738b964937eea2d3e0/WORKSPACE#L28-L32
-        URLS = ["https://storage.googleapis.com/rbe-toolchain/bazel-configs/rbe-ubuntu1604/latest/rbe_default.tar"],
+        URLS = ["https://github.com/bazelbuild/continuous-integration/releases/download/rules-{version}/bazelci_rules-{version}.tar.gz"],
     ),
     PKG = version(
-        version = "0.7.0",
+        version = "1.0.1",
         url_templates = [
             "https://github.com/bazelbuild/rules_pkg/releases/download/{version}/rules_pkg-{version}.tar.gz",
         ],
-        sha256 = "8a298e832762eda1830597d64fe7db58178aa84cd5926d76d5b744d6558941c2",
+        sha256 = "d20c951960ed77cb7b341c2a59488534e494d5ad1d30c4818c736d57772a9fef",
     ),
     # needed for rules_pkg and java
     RULES_KOTLIN = version(
@@ -126,11 +124,11 @@ versions = struct(
         sha256 = None,
     ),
     RULES_TESTING = version(
-        version = "0.5.0",
+        version = "0.6.0",
         url_templates = [
             "https://github.com/bazelbuild/rules_testing/releases/download/v{version}/rules_testing-v{version}.tar.gz",
         ],
-        sha256 = "b84ed8546f1969d700ead4546de9f7637e0f058d835e47e865dcbb13c4210aed",
+        sha256 = "02c62574631876a4e3b02a1820cb51167bb9cdcdea2381b2fa9d9b8b11c407c4",
     ),
     KOTLINX_SERIALIZATION_CORE_JVM = version(
         version = "1.6.3",
@@ -153,5 +151,25 @@ versions = struct(
         ],
         sha256 = "d3234179bcff1886d53d67c11eca47f7f3cf7b63c349d16965f6db51b7f3dd9a",
     ),
+    RULES_BAZEL_INTEGRATION_TEST = version(
+        version = "0.26.0",
+        url_templates = [
+            "https://github.com/bazel-contrib/rules_bazel_integration_test/releases/download/v{version}/rules_bazel_integration_test.v{version}.tar.gz",
+        ],
+        sha256 = "ab56cdd55a28781287242c7124ce9ff791ae8318ed641057f10edd98c55d7ed5",
+    ),
+    CGRINDEL_BAZEL_STARLIB = version(
+        version = "0.21.0",
+        sha256 = "43e375213dabe0c3928e65412ea7ec16850db93285c8c6f8b0eaa41cacd0f882",
+        url_templates = [
+            "https://github.com/cgrindel/bazel-starlib/releases/download/v{version}/bazel-starlib.v{version}.tar.gz",
+        ],
+    ),
+    RULES_CC = version(
+        version = "0.0.9",
+        url_templates = ["https://github.com/bazelbuild/rules_cc/releases/download/{version}/rules_cc-{version}.tar.gz"],
+        sha256 = "2037875b9a4456dce4a79d112a8ae885bbc4aad968e6587dca6e64f3a0900cdf",
+        strip_prefix_template = "rules_cc-{version}",
+    ),
     use_repository = _use_repository,
 )