Verify that the list of Cargo.tomls to vendor matches bootstrap/dist.rs

When these change upstream we need to change our build_rust.py script
so that we don't get missing vendored dependencies (which can look like
wrong versions and gets confusing).

[email protected]

Bug: 1441917
Change-Id: Ia8b70a206ef1c7c7de198161002655e7392fa9c9
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/4508134
Commit-Queue: danakj <[email protected]>
Reviewed-by: Collin Baker <[email protected]>
Cr-Commit-Position: refs/heads/main@{#1140231}
NOKEYCHECK=True
GitOrigin-RevId: 12a7862a280dbb36a57c5e6f38c4a21f3c77ea6c

Author: danakj

build_rust.py

@@ -39,6 +39,7 @@
 import json
 import platform
 import os
+import re
 import shutil
 import string
 import subprocess
@@ -97,6 +98,8 @@
                 'github.com/rust-lang/rust')
 
 RUST_SRC_DIR = os.path.join(THIRD_PARTY_DIR, 'rust_src', 'src')
+RUST_BOOTSTRAP_DIST_RS = os.path.join(RUST_SRC_DIR, 'src', 'bootstrap',
+                                      'dist.rs')
 STAGE0_JSON_PATH = os.path.join(RUST_SRC_DIR, 'src', 'stage0.json')
 # Download crates.io dependencies to rust-src subdir (rather than $HOME/.cargo)
 CARGO_HOME_DIR = os.path.join(RUST_SRC_DIR, 'cargo-home')
@@ -256,6 +259,42 @@ def InstallBetaPackage(package_dir, install_dir):
 
 def CargoVendor(cargo_bin):
     '''Runs `cargo vendor` to pull down dependencies.'''
+    # From https://github.com/rust-lang/rust/blob/4a18324a4df6bc98bec0b54d35908d7a9cdc7c32/src/bootstrap/dist.rs#L1008-L1015:
+    # The additional `--sync` Cargo.toml files are not part of the top level
+    # workspace.
+    SYNC_TARGETS = [
+        './src/tools/cargo/Cargo.toml',
+        './src/tools/rust-analyzer/Cargo.toml',
+        './compiler/rustc_codegen_cranelift/Cargo.toml',
+        './src/bootstrap/Cargo.toml',
+    ]
+
+    # Try to verify our sync targets match the upstream nightly tarball
+    # builder's.
+    BUILDER_REGEX = (r'(?:'
+                     r'\s*\.arg\("--sync"\)'
+                     r'\s*\.arg\(builder.src.join\("(?P<target>[^"]+)"\)\)'
+                     r')')
+    content = ''
+    with open(RUST_BOOTSTRAP_DIST_RS) as f:
+        content = ''.join([line.strip() for line in f])
+    upstream_sync_targets = re.compile(BUILDER_REGEX).findall(content)
+    error = False
+    for s in SYNC_TARGETS:
+        if not s in upstream_sync_targets:
+            print(f'Upstream bootstrap/dist.rs removed "--sync {s}", '
+                  'so it should be removed from SYNC_TARGETS in '
+                  '//tools/rust/build_rust.py.')
+            error = True
+    for s in upstream_sync_targets:
+        if not s in SYNC_TARGETS:
+            print(f'Upstream bootstrap/dist.rs added "--sync {s}", '
+                  'so it should be added to SYNC_TARGETS in '
+                  '//tools/rust/build_rust.py.')
+            error = True
+    if error:
+        sys.exit(1)
+
     os.chdir(RUST_SRC_DIR)
 
     for i in range(0, 3):
@@ -273,23 +312,14 @@ def CargoVendor(cargo_bin):
         else:
             sys.exit(1)
 
-        # From https://github.com/rust-lang/rust/blob/master/src/bootstrap/dist.rs#L1003-L1013:
-        # The additional `--sync` Cargo.toml files are not part of the top level
-        # workspace.
         vendor_cmd = [
             cargo_bin,
             'vendor',
             '--locked',
             '--versioned-dirs',
-            '--sync',
-            'src/tools/cargo/Cargo.toml',
-            '--sync',
-            'src/tools/rust-analyzer/Cargo.toml',
-            '--sync',
-            'compiler/rustc_codegen_cranelift/Cargo.toml',
-            '--sync',
-            'src/bootstrap/Cargo.toml',
         ]
+        for s in SYNC_TARGETS:
+            vendor_cmd.extend(['--sync', s])
         if RunCommand(vendor_cmd, fail_hard=False):
             break  # Success, break out of the retry loop.
         elif i < 2:

update_rust.py

@@ -192,7 +192,6 @@ def main():
         shutil.rmtree(RUST_TOOLCHAIN_OUT_DIR)
 
     try:
-        platform_prefix = GetPlatformUrlPrefix(GetDefaultHostOs())
         DownloadAndUnpack(url, THIRD_PARTY_DIR)
     except urllib.error.HTTPError as e:
         print(f'error: Failed to download Rust package')