Rollup merge of rust-lang#138533 - Kobzol:try-job-auto-tests, r=marcoieni

Only use `DIST_TRY_BUILD` for try jobs that were not selected explicitly

Some CI jobs (x64 Linux, ARM64 Linux and x64 MSVC) use the `opt-dist` tool to build an optimized toolchain using PGO and BOLT. When performing a default try build for x64 Linux, in most cases we want to run perf. on that artifact. To reduce the latency of this common use-case, `opt-dist` skips building several components not needed for perf., and it also skips running post-optimization tests, when it detects that the job is executed as a try job (not a merge/auto job).

This is useful, but it also means that if you *want* to run the tests, you had to go to `jobs.yml` and manually comment this environment variable, create a WIP commit, do a try build, and then remove the WIP commit, which is annoying (in the similar way that modifying what gets run in try builds was annoying before we had the `try-job` annotations).

I thought that we could introduce some additional PR description marker like `try-job-run-tests`, but it's hard to discover that such things exist.

Instead, I think that there's a much simpler heuristic for determining whether `DIST_TRY_BUILD` should be used (that I implemented in this PR):
- If you do just ``@bors` try`, without any custom try jobs selected, `DIST_TRY_BUILD` will be activated, to finish the build as fast as possible.
- If you specify any custom try jobs, you are most likely doing experiments and you want to see if tests pass and everything builds as it should. The `DIST_TRY_BUILD` variable will thus *not* be set in this case.

In this way, if you want to run dist tests, you can just add the `try-job: dist-x86_64-linux` line to the PR description, and you don't need to create any WIP commits.

r? `@marcoieni`

Author: matthiaskrgr

src/ci/citool/src/jobs.rs

@@ -185,6 +185,20 @@ fn calculate_jobs(
             env.extend(crate::yaml_map_to_json(&job.env));
             let full_name = format!("{prefix} - {}", job.name);
 
+            // When the default `@bors try` job is executed (which is usually done
+            // for benchmarking performance, running crater or for downloading the
+            // built toolchain using `rustup-toolchain-install-master`),
+            // we inject the `DIST_TRY_BUILD` environment variable to the jobs
+            // to tell `opt-dist` to make the build faster by skipping certain steps.
+            if let RunType::TryJob { job_patterns } = run_type {
+                if job_patterns.is_none() {
+                    env.insert(
+                        "DIST_TRY_BUILD".to_string(),
+                        serde_json::value::Value::Number(1.into()),
+                    );
+                }
+            }
+
             GithubActionsJob {
                 name: job.name,
                 full_name,

src/ci/citool/tests/jobs.rs

@@ -14,10 +14,10 @@ fn auto_jobs() {
 #[test]
 fn try_jobs() {
     let stdout = get_matrix("push", "commit", "refs/heads/try");
-    insta::assert_snapshot!(stdout, @r#"
+    insta::assert_snapshot!(stdout, @r###"
     jobs=[{"name":"dist-x86_64-linux","full_name":"try - dist-x86_64-linux","os":"ubuntu-22.04-16core-64gb","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","CODEGEN_BACKENDS":"llvm,cranelift","DEPLOY_BUCKET":"rust-lang-ci2","DIST_TRY_BUILD":1,"TOOLSTATE_PUBLISH":1}}]
     run_type=try
-    "#);
+    "###);
 }
 
 #[test]
@@ -30,10 +30,10 @@ try-job: aarch64-gnu
 try-job: dist-i686-msvc"#,
         "refs/heads/try",
     );
-    insta::assert_snapshot!(stdout, @r#"
-    jobs=[{"name":"aarch64-gnu","full_name":"try - aarch64-gnu","os":"ubuntu-22.04-arm","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","DEPLOY_BUCKET":"rust-lang-ci2","DIST_TRY_BUILD":1,"TOOLSTATE_PUBLISH":1},"free_disk":true},{"name":"dist-i686-msvc","full_name":"try - dist-i686-msvc","os":"windows-2022","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","CODEGEN_BACKENDS":"llvm,cranelift","DEPLOY_BUCKET":"rust-lang-ci2","DIST_REQUIRE_ALL_TOOLS":1,"DIST_TRY_BUILD":1,"RUST_CONFIGURE_ARGS":"--build=i686-pc-windows-msvc --host=i686-pc-windows-msvc --target=i686-pc-windows-msvc,i586-pc-windows-msvc --enable-full-tools --enable-profiler","SCRIPT":"python x.py dist bootstrap --include-default-paths","TOOLSTATE_PUBLISH":1}}]
+    insta::assert_snapshot!(stdout, @r###"
+    jobs=[{"name":"aarch64-gnu","full_name":"try - aarch64-gnu","os":"ubuntu-22.04-arm","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","DEPLOY_BUCKET":"rust-lang-ci2","TOOLSTATE_PUBLISH":1},"free_disk":true},{"name":"dist-i686-msvc","full_name":"try - dist-i686-msvc","os":"windows-2022","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","CODEGEN_BACKENDS":"llvm,cranelift","DEPLOY_BUCKET":"rust-lang-ci2","DIST_REQUIRE_ALL_TOOLS":1,"RUST_CONFIGURE_ARGS":"--build=i686-pc-windows-msvc --host=i686-pc-windows-msvc --target=i686-pc-windows-msvc,i586-pc-windows-msvc --enable-full-tools --enable-profiler","SCRIPT":"python x.py dist bootstrap --include-default-paths","TOOLSTATE_PUBLISH":1}}]
     run_type=try
-    "#);
+    "###);
 }
 
 #[test]

src/ci/citool/tests/test-jobs.yml

@@ -53,13 +53,6 @@ envs:
 
   try:
     <<: *production
-    # The following env var activates faster `try` builds in `opt-dist` by, e.g.
-    # - building only the more commonly useful components (we rarely need e.g. rust-docs in try
-    #   builds)
-    # - not running `opt-dist`'s post-optimization smoke tests on the resulting toolchain
-    #
-    # If you *want* these to happen however, temporarily comment it before triggering a try build.
-    DIST_TRY_BUILD: 1
 
   auto:
     <<: *production

src/ci/github-actions/jobs.yml

@@ -82,15 +82,13 @@ envs:
     AWS_REGION: us-west-1
     TOOLSTATE_PUBLISH: 1
 
+  # Try builds started through `@bors try` (without specifying custom jobs
+  # in the PR description) will be passed the `DIST_TRY_BUILD` environment
+  # variable by citool.
+  # This tells the `opt-dist` tool to skip building certain components
+  # and skip running tests, so that the try build finishes faster.
   try:
     <<: *production
-    # The following env var activates faster `try` builds in `opt-dist` by, e.g.
-    # - building only the more commonly useful components (we rarely need e.g. rust-docs in try
-    #   builds)
-    # - not running `opt-dist`'s post-optimization smoke tests on the resulting toolchain
-    #
-    # If you *want* these to happen however, temporarily comment it before triggering a try build.
-    DIST_TRY_BUILD: 1
 
   auto:
     <<: *production

src/doc/rustc-dev-guide/src/tests/ci.md

@@ -180,6 +180,8 @@ their results can be seen [here](https://github.com/rust-lang-ci/rust/actions),
 although usually you will be notified of the result by a comment made by bors on
 the corresponding PR.
 
+Note that if you start the default try job using `@bors try`, it will skip building several `dist` components and running post-optimization tests, to make the build duration shorter. If you want to execute the full build as it would happen before a merge, add an explicit `try-job` pattern with the name of the default try job (currently `dist-x86_64-linux`).
+
 Multiple try builds can execute concurrently across different PRs.
 
 <div class="warning">