Pull rust compiler sources from Git and ship them with the toolchain

Working with tarballs makes it harder to sheriff as you can't specify
a git hash to work with.

Shipping the sources with the toolchain ensures devs can build the
correct stdlib for the compiler at all times.

Bug: 1401042
Change-Id: I5094016392d268aafe746d8852a7544fc8d11f4f
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/4190357
Reviewed-by: Hans Wennborg <[email protected]>
Commit-Queue: danakj <[email protected]>
Reviewed-by: Collin Baker <[email protected]>
Cr-Commit-Position: refs/heads/main@{#1097651}
NOKEYCHECK=True
GitOrigin-RevId: fed1e1b696e015961c7ddf834c4ce0623e55cafd

Author: danakj

README.md

@@ -301,3 +301,18 @@ Crubit is built on the bots, the tests are commented out in
 `//build/rust/tests/BUILD.gn`, but they should still be built and run before
 rolling Crubit.  TODO(https://crbug.com/1329611): Rephrase this paragraph
 after Crubit is built and tested on the bots.
+
+## Possible Failures
+
+### Missing dependencies
+
+`build_rust.py` will vendor all dependencies before starting the build. To do
+this it first initializes git submodules. Then it runs `cargo vendor`. However
+some parts of the compiler build are excluded from the top level Cargo.toml
+workspace. Thus it passes `--sync dir` for a number of subdirectories, based
+on [dist.rs, the nightly tarball packager](
+https://github.com/rust-lang/rust/blob/master/src/bootstrap/dist.rs#L986-L995).
+
+If another Cargo.toml is required in the future, and not part of the workspace
+it would produce missing dependencies, and the set of directories in
+`build_rust.py` would need to be updated.

build_rust.py

@@ -33,15 +33,18 @@
 '''
 
 import argparse
+import base64
 import collections
 import hashlib
+import json
 import platform
 import os
 import pipes
 import shutil
 import string
 import subprocess
 import sys
+import urllib
 
 from pathlib import Path
 
@@ -51,23 +54,25 @@
                  'scripts'))
 
 from build import (AddCMakeToPath, AddOpenSSLToEnv, AddZlibToPath,
-                   GetLibXml2Dirs, RunCommand)
-from update import (CLANG_REVISION, CLANG_SUB_REVISION, LLVM_BUILD_DIR,
-                    GetDefaultHostOs, RmTree, UpdatePackage)
+                   CheckoutGitRepo, GetLibXml2Dirs, LLVM_BUILD_TOOLS_DIR,
+                   RunCommand)
+from update import (CLANG_REVISION, CLANG_SUB_REVISION, DownloadAndUnpack,
+                    LLVM_BUILD_DIR, GetDefaultHostOs, RmTree, UpdatePackage)
 import build
 
 from update_rust import (CHROMIUM_DIR, RUST_REVISION, RUST_SUB_REVISION,
                          RUST_TOOLCHAIN_OUT_DIR, STAGE0_JSON_SHA256,
                          THIRD_PARTY_DIR, VERSION_STAMP_PATH,
                          GetPackageVersionForBuild)
 
-RUST_GIT_URL = 'https://github.com/rust-lang/rust/'
+RUST_GIT_URL = ('https://chromium.googlesource.com/external/' +
+                'github.com/rust-lang/rust')
 
 RUST_SRC_DIR = os.path.join(THIRD_PARTY_DIR, 'rust_src', 'src')
 STAGE0_JSON_PATH = os.path.join(RUST_SRC_DIR, 'src', 'stage0.json')
 # Download crates.io dependencies to rust-src subdir (rather than $HOME/.cargo)
 CARGO_HOME_DIR = os.path.join(RUST_SRC_DIR, 'cargo-home')
-RUST_SRC_VERSION_FILE_PATH = os.path.join(RUST_SRC_DIR, 'version')
+RUST_SRC_VERSION_FILE_PATH = os.path.join(RUST_SRC_DIR, 'src', 'version')
 RUST_SRC_GIT_COMMIT_INFO_FILE_PATH = os.path.join(RUST_SRC_DIR,
                                                   'git-commit-info')
 RUST_TOOLCHAIN_LIB_DIR = os.path.join(RUST_TOOLCHAIN_OUT_DIR, 'lib')
@@ -77,6 +82,8 @@
                                                   'vendor')
 RUST_CONFIG_TEMPLATE_PATH = os.path.join(
     os.path.dirname(os.path.abspath(__file__)), 'config.toml.template')
+RUST_CARGO_CONFIG_TEMPLATE_PATH = os.path.join(
+    os.path.dirname(os.path.abspath(__file__)), 'cargo-config.toml.template')
 RUST_SRC_VENDOR_DIR = os.path.join(RUST_SRC_DIR, 'vendor')
 
 if sys.platform == 'win32':
@@ -93,8 +100,8 @@
 # Which test suites to run. Any failure will fail the build.
 TEST_SUITES = [
     'library/std',
-    'src/test/codegen',
-    'src/test/ui',
+    'tests/codegen',
+    'tests/ui',
 ]
 
 EXCLUDED_TESTS = [
@@ -142,6 +149,82 @@ def quote_string(s: str):
         output.write(template.substitute(subs))
 
 
+def FetchCargo(rust_git_hash):
+    '''Downloads the beta Cargo package specified for the compiler build
+
+    Unpacks the package and returns the path to the binary itself.
+    '''
+    if sys.platform == 'win32':
+        target = 'cargo-beta-x86_64-pc-windows-msvc'
+    elif sys.platform == 'darwin':
+        if platform.machine() == 'arm64':
+            target = 'rust-std-beta-aarch64-apple-darwin'
+        else:
+            target = 'cargo-beta-x86_64-apple-darwin'
+    else:
+        target = 'cargo-beta-x86_64-unknown-linux-gnu'
+
+    # Pull the stage0 JSON to find the Cargo binary intended to be used to
+    # build this version of the Rust compiler.
+    STAGE0_JSON_URL = (
+        'https://chromium.googlesource.com/external/github.com/'
+        'rust-lang/rust/+/{GIT_HASH}/src/stage0.json?format=TEXT')
+    base64_text = urllib.request.urlopen(
+        STAGE0_JSON_URL.format(GIT_HASH=rust_git_hash)).read().decode("utf-8")
+    stage0 = json.loads(base64.b64decode(base64_text))
+
+    # The stage0 JSON contains the path to all tarballs it uses binaries from,
+    # including cargo.
+    for k in stage0['checksums_sha256'].keys():
+        if k.endswith(target + '.tar.gz'):
+            cargo_tgz = k
+
+    server = stage0['config']['dist_server']
+    DownloadAndUnpack(f'{server}/{cargo_tgz}', LLVM_BUILD_TOOLS_DIR)
+
+    bin_path = os.path.join(LLVM_BUILD_TOOLS_DIR, target, 'cargo', 'bin')
+    if sys.platform == 'win32':
+        return os.path.join(bin_path, 'cargo.exe')
+    else:
+        return os.path.join(bin_path, 'cargo')
+
+
+def CargoVendor(cargo_bin):
+    '''Runs `cargo vendor` to pull down dependencies.'''
+    os.chdir(RUST_SRC_DIR)
+
+    # Some Submodules are part of the workspace and need to exist (so we can
+    # read their Cargo.toml files) before we can vendor their deps.
+    RunCommand([
+        'git', 'submodule', 'update', '--init', '--recursive', '--depth', '1'
+    ])
+
+    # From https://github.com/rust-lang/rust/blob/master/src/bootstrap/dist.rs#L986-L995
+    # The additional `--sync` Cargo.toml files are not part of the top level
+    # workspace.
+    RunCommand([
+        cargo_bin,
+        'vendor',
+        '--locked',
+        '--versioned-dirs',
+        '--sync',
+        'src/tools/rust-analyzer/Cargo.toml',
+        '--sync',
+        'compiler/rustc_codegen_cranelift/Cargo.toml',
+        '--sync',
+        'src/bootstrap/Cargo.toml',
+    ])
+
+    # Make a `.cargo/config.toml` the points to the `vendor` directory for all
+    # dependency crates.
+    try:
+        os.mkdir(os.path.join(RUST_SRC_DIR, '.cargo'))
+    except FileExistsError:
+        pass
+    shutil.copyfile(RUST_CARGO_CONFIG_TEMPLATE_PATH,
+                    os.path.join(RUST_SRC_DIR, '.cargo', 'config.toml'))
+
+
 def RunXPy(sub, args, llvm_bins_path, zlib_path, libxml2_dirs, build_mac_arm,
            gcc_toolchain_path, verbose):
     ''' Run x.py, Rust's build script'''
@@ -152,6 +235,7 @@ def RunXPy(sub, args, llvm_bins_path, zlib_path, libxml2_dirs, build_mac_arm,
         'LDFLAGS',
         'RUSTFLAGS_BOOTSTRAP',
         'RUSTFLAGS_NOT_BOOTSTRAP',
+        'RUSTDOCFLAGS',
     ]
 
     RUSTENV = collections.defaultdict(str, os.environ)
@@ -220,6 +304,10 @@ def RunXPy(sub, args, llvm_bins_path, zlib_path, libxml2_dirs, build_mac_arm,
         RUSTENV['RUSTFLAGS_NOT_BOOTSTRAP'] += (
             f' -L native={gcc_toolchain_path}/lib64')
 
+    # Rustdoc should use our clang linker as well, as we pass flags that
+    # the system linker may not understand.
+    RUSTENV['RUSTDOCFLAGS'] += f' -Clinker={RUSTENV["CC"]}'
+
     # Cargo normally stores files in $HOME. Override this.
     RUSTENV['CARGO_HOME'] = CARGO_HOME_DIR
 
@@ -243,16 +331,27 @@ def GetTestArgs():
     return args
 
 
-def GetVersionStamp():
-    # We must generate a version stamp that contains the expected `rustc
-    # --version` output. This contains the Rust release version, git commit data
-    # that the nightly tarball was generated from, and chromium-specific package
-    # information.
+def MakeVersionStamp(git_hash):
+    # We must generate a version stamp that contains the full version of the
+    # built Rust compiler:
+    # * The version number returned from `rustc --version`.
+    # * The git hash.
+    # * The chromium revision name of the compiler build, which includes the
+    #   associated clang/llvm version.
     with open(RUST_SRC_VERSION_FILE_PATH) as version_file:
         rust_version = version_file.readline().rstrip()
+    return (f'rustc {rust_version} {git_hash}'
+            f' ({GetPackageVersionForBuild()} chromium)\n')
+
 
-    return ('rustc %s (%s chromium)\n' %
-            (rust_version, GetPackageVersionForBuild()))
+def GetLatestRustCommit():
+    """Get the latest commit hash in the LLVM monorepo."""
+    main = json.loads(
+        urllib.request.urlopen('https://chromium.googlesource.com/external/' +
+                               'github.com/rust-lang/rust/' +
+                               '+/refs/heads/main?format=JSON').read().decode(
+                                   "utf-8").replace(")]}'", ""))
+    return main['commit']
 
 
 def main():
@@ -271,6 +370,9 @@ def main():
         help=
         'checkout Rust, verify the stage0 hash, then quit without building. '
         'Will print the actual hash if different than expected.')
+    parser.add_argument('--skip-checkout',
+                        action='store_true',
+                        help='do not create or update any checkouts')
     parser.add_argument('--skip-clean',
                         action='store_true',
                         help='skip x.py clean step')
@@ -280,6 +382,9 @@ def main():
     parser.add_argument('--skip-install',
                         action='store_true',
                         help='do not install to RUST_TOOLCHAIN_OUT_DIR')
+    parser.add_argument('--rust-force-head-revision',
+                        action='store_true',
+                        help='build the latest revision')
     parser.add_argument(
         '--fetch-llvm-libs',
         action='store_true',
@@ -324,6 +429,14 @@ def main():
     else:
         llvm_bins_path = os.path.join(build.LLVM_BOOTSTRAP_DIR, 'bin')
 
+    if args.rust_force_head_revision:
+        checkout_revision = GetLatestRustCommit()
+    else:
+        checkout_revision = RUST_REVISION
+
+    if not args.skip_checkout:
+        CheckoutGitRepo('Rust', RUST_GIT_URL, checkout_revision, RUST_SRC_DIR)
+
     VerifyStage0JsonHash()
     if args.verify_stage0_hash:
         # The above function exits and prints the actual hash if verification
@@ -344,6 +457,9 @@ def main():
     # Set up config.toml in Rust source tree to configure build.
     Configure(llvm_bins_path, llvm_libs_root)
 
+    cargo_bin = FetchCargo(checkout_revision)
+    CargoVendor(cargo_bin)
+
     AddCMakeToPath()
 
     # Require zlib compression.
@@ -404,19 +520,18 @@ def main():
         shutil.rmtree(RUST_TOOLCHAIN_OUT_DIR)
 
     RunXPy('install', DISTRIBUTION_ARTIFACTS, llvm_bins_path, zlib_path,
-           libxml2_dirs, args.gcc_toolchain, args.verbose)
+           libxml2_dirs, args.build_mac_arm, args.gcc_toolchain, args.verbose)
+
+    # Copy additional sources required for building stdlib out of
+    # RUST_TOOLCHAIN_SRC_DIST_DIR.
+    print(f'Copying vendored dependencies to {RUST_TOOLCHAIN_OUT_DIR} ...')
+    shutil.copytree(RUST_SRC_VENDOR_DIR, RUST_TOOLCHAIN_SRC_DIST_VENDOR_DIR)
 
     with open(VERSION_STAMP_PATH, 'w') as stamp:
-        stamp.write(GetVersionStamp())
+        stamp.write(MakeVersionStamp(checkout_revision))
 
     return 0
 
-    # TODO(crbug.com/1342708): fix vendoring and re-enable.
-    # x.py installed library sources to our toolchain directory. We also need to
-    # copy the vendor directory so Chromium checkouts have all the deps needed
-    # to build std.
-    # shutil.copytree(RUST_SRC_VENDOR_DIR, RUST_TOOLCHAIN_SRC_DIST_VENDOR_DIR)
-
 
 if __name__ == '__main__':
     sys.exit(main())

cargo-config.toml.template

@@ -0,0 +1,11 @@
+# Copyright 2022 The Chromium Authors
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# This file is placed in `.cargo/config/` when building rustc.
+
+[source.crates-io]
+replace-with = "vendored-sources"
+
+[source.vendored-sources]
+directory = "vendor"

config.toml.template

@@ -18,17 +18,15 @@ channel = "nightly"
 description = "$PACKAGE_VERSION chromium"
 
 [build]
-# Vendor crates.io dependencies to rust-src/vendor and check they match root
-# Cargo.lock. In the future we will probably want to vendor these ourself like
-# we do with third_party/rust dependencies. For now, the Rust build script auto
-# fetches the dependencies.
-#
-# This is added now to work around a permissions bug in Rust's bootstrap:
-# it fails to run under sudo even if the sudo user is not root, unless sources
-# are vendored in tree.
-locked-deps = true
+# We pull all dependencies into the src/vendor/ directory.
 vendor = true
 
+# Check that vendored dependencies match the root Cargo.lock.
+locked-deps = true
+
+# Don't build documentation for the stdlib.
+docs = false
+
 # We enable profiler so that PGO and coverage build options are available
 # with the rust compiler we build.
 profiler = true

update_rust.py

@@ -29,8 +29,7 @@
 
 # These fields are written by //tools/clang/scripts/upload_revision.py, and
 # should not be changed manually.
-RUST_REVISION_TAG = '2'
-RUST_REVISION = '20221209'
+RUST_REVISION = 'c8e6a9e8b6251bbc8276cb78cabe1998deecbed7'
 RUST_SUB_REVISION = 1
 
 # Trunk on 2022-10-15.
@@ -98,8 +97,7 @@ def GetStampVersion():
     if os.path.exists(RUST_TOOLCHAIN_OUT_DIR):
         with open(VERSION_STAMP_PATH) as version_file:
             existing_stamp = version_file.readline().rstrip()
-        version_re = re.compile(
-            r'rustc [0-9.]+-nightly \([0-9a-f -]+\) \((.+?) chromium\)')
+        version_re = re.compile(r'rustc [0-9.]+ [0-9a-f]+ \((.+?) chromium\)')
         match = version_re.fullmatch(existing_stamp)
         if match is None:
             return None