Cleanup bootstrap scripts (#6117)

* Remove github latest-release downloader, since that's a hermeticity
issue that can cause nondeterminism as new github versions are released,
or even as time passes and you run the command again
* Remove `No mill version specified` warning, since now we recommend
people to rely on the `MILL_DEFAULT_VERSION` in the docs
* Remove check for system-installed Mill, which adds a bunch of
complexity just to save a single download
* Remove header comment that is already out of date and impossible to
maintain
* DRY up the `sed` regexes in `mill.sh` and the `sed`-emulation code in
`mill.bat`
* Re-arranged the two scripts and standardized variable names so the
logic happens in the same order, so they are easier to read through
side-by-side and compare
* Move mill temp download to `out/mill-bootstrap-download` to fix
#5837
* Added a `dist.scripts.test` windows CI job to ensure the windows code
paths are covered
* Tweaked a few windows cache paths to fix
#6109

Covered by unit tests, and tested manually on my macbook pro and windows
microsoft surface laptop 7

Author: lihaoyi

.github/workflows/run-tests.yml

@@ -216,6 +216,9 @@ jobs:
           - java-version: 17
             millargs: '"example.scalalib.{basic,publishing}.__.packaged.daemon"'
 
+          - java-version: 17
+            millargs: '"dist.scripts.test"' # run this on both windows and mac to exercise bootstrap scripts
+
           - java-version: 17
             millargs: "'integration.failure.__.packaged.nodaemon'"
 

dist/scripts/src/mill.bat

@@ -1,37 +1,5 @@
 @echo off
 
-rem This is a wrapper script, that automatically selects or downloads Mill from Maven Central or GitHub release pages.
-rem
-rem This script determines the Mill version to use by trying these sources
-rem   - env-variable `MILL_VERSION`
-rem   - local file `.mill-version`
-rem   - local file `.config/mill-version`
-rem   - `mill-version` from YAML frontmatter of current buildfile
-rem   - if accessible, find the latest stable version available on Maven Central ({{{ mill-maven-url }}})
-rem   - env-variable `DEFAULT_MILL_VERSION`
-rem
-rem If a version has the suffix '-native' a native binary will be used.
-rem If a version has the suffix '-jvm' an executable jar file will be used, requiring an already installed Java runtime.
-rem If no such suffix is found, the script will pick a default based on version and platform.
-rem
-rem Once a version was determined, it tries to use either
-rem    - a system-installed mill, if found and it's version matches
-rem    - an already downloaded version under {{{ mill-download-cache-win }}}
-rem
-rem If no working mill version was found on the system,
-rem this script downloads a binary file from Maven Central or Github Pages (this is version dependent)
-rem into a cache location ({{{ mill-download-cache-win }}}).
-rem
-rem Mill Project URL: {{{ mill-repo-url }}}
-rem Script Version: {{{ mill-version }}}
-rem
-rem If you want to improve this script, please also contribute your changes back!
-rem This script was generated from: {{{ template-file }}}
-rem
-rem Licensed under the Apache License, Version 2.0
-
-rem setlocal seems to be unavailable on Windows 95/98/ME
-rem but I don't think we need to support them in 2019
 setlocal enabledelayedexpansion
 
 if [!DEFAULT_MILL_VERSION!]==[] ( set "DEFAULT_MILL_VERSION={{{ mill-version }}}" )
@@ -65,9 +33,24 @@ if [!MILL_VERSION!]==[] (
     if exist .config\mill-version (
       set /p MILL_VERSION=<.config\mill-version
     ) else (
+      rem Determine which config file to use for version extraction
+      set "MILL_VERSION_CONFIG_FILE="
+      set "MILL_VERSION_SEARCH_PATTERN="
+
       if exist build.mill.yaml (
+        set "MILL_VERSION_CONFIG_FILE=build.mill.yaml"
+        set "MILL_VERSION_SEARCH_PATTERN=mill-version:"
+      ) else (
+        if not "%MILL_BUILD_SCRIPT%"=="" (
+          set "MILL_VERSION_CONFIG_FILE=%MILL_BUILD_SCRIPT%"
+          set "MILL_VERSION_SEARCH_PATTERN=//\|.*mill-version"
+        )
+      )
+
+      rem Process the config file if found
+      if not "!MILL_VERSION_CONFIG_FILE!"=="" (
         rem Find the line and process it
-        for /f "tokens=*" %%a in ('findstr /R /C:"mill-version:" "build.mill.yaml"') do (
+        for /f "tokens=*" %%a in ('findstr /R /C:"!MILL_VERSION_SEARCH_PATTERN!" "!MILL_VERSION_CONFIG_FILE!"') do (
             set "line=%%a"
 
             rem --- 1. Replicate sed 's/.*://' ---
@@ -85,7 +68,7 @@ if [!MILL_VERSION!]==[] (
             set "line=!line:'=!"
             set "line=!line:"=!"
 
-            rem --- 4. NEW: Replicate sed's trim/space removal ---
+            rem --- 4. Replicate sed's trim/space removal ---
             rem Remove all space characters from the result. This is more robust.
             set "MILL_VERSION=!line: =!"
 
@@ -95,59 +78,23 @@ if [!MILL_VERSION!]==[] (
 
         :version_found
         rem no-op
-      ) else (
-        if not "%MILL_BUILD_SCRIPT%"=="" (
-          rem Find the line and process it
-          for /f "tokens=*" %%a in ('findstr /R /C:"//\|.*mill-version" "%MILL_BUILD_SCRIPT%"') do (
-              set "line=%%a"
-
-              rem --- 1. Replicate sed 's/.*://' ---
-              rem This removes everything up to and including the first colon
-              set "line=!line:*:=!"
-
-              rem --- 2. Replicate sed 's/#.*//' ---
-              rem Split on '#' and keep the first part
-              for /f "tokens=1 delims=#" %%b in ("!line!") do (
-                  set "line=%%b"
-              )
-
-              rem --- 3. Replicate sed 's/['"]//g' ---
-              rem Remove all quotes
-              set "line=!line:'=!"
-              set "line=!line:"=!"
-
-              rem --- 4. NEW: Replicate sed's trim/space removal ---
-              rem Remove all space characters from the result. This is more robust.
-              set "MILL_VERSION=!line: =!"
-
-              rem We found the version, so we can exit the loop
-              goto :version_found
-          )
-
-          :version_found
-          rem no-op
-        ) else (
-          rem no-op
-        )
       )
     )
   )
 )
 
 if [!MILL_VERSION!]==[] (
-    echo No mill version specified. >&2
-    echo You should provide a version via a '//^| mill-version: ' comment or a '.mill-version' file. >&2
     set MILL_VERSION=%DEFAULT_MILL_VERSION%
 )
 
-if [!MILL_DOWNLOAD_PATH!]==[] set MILL_DOWNLOAD_PATH=%USERPROFILE%\.mill\download
+if [!MILL_FINAL_DOWNLOAD_FOLDER!]==[] set MILL_FINAL_DOWNLOAD_FOLDER=%USERPROFILE%\.cache\mill\download
 
 rem without bat file extension, cmd doesn't seem to be able to run it
 
 set "MILL_NATIVE_SUFFIX=-native"
 set "MILL_JVM_SUFFIX=-jvm"
-set "FULL_MILL_VERSION=%MILL_VERSION%"
-set "MILL_EXT=.bat"
+set "MILL_FULL_VERSION=%MILL_VERSION%"
+set "MILL_DOWNLOAD_EXT=.bat"
 set "ARTIFACT_SUFFIX="
 REM Check if MILL_VERSION contains MILL_NATIVE_SUFFIX
 echo !MILL_VERSION! | findstr /C:"%MILL_NATIVE_SUFFIX%" >nul
@@ -157,7 +104,7 @@ if !errorlevel! equ 0 (
     REM https://github.com/oracle/graal/issues/9215
     IF /I NOT "%PROCESSOR_ARCHITECTURE%"=="ARM64" (
         set "ARTIFACT_SUFFIX=-native-windows-amd64"
-        set "MILL_EXT=.exe"
+        set "MILL_DOWNLOAD_EXT=.exe"
     ) else (
         rem no-op
     )
@@ -185,15 +132,15 @@ if !errorlevel! equ 0 (
         if "!SKIP_VERSION!"=="false" (
             IF /I NOT "%PROCESSOR_ARCHITECTURE%"=="ARM64" (
                 set "ARTIFACT_SUFFIX=-native-windows-amd64"
-                set "MILL_EXT=.exe"
+                set "MILL_DOWNLOAD_EXT=.exe"
             )
         ) else (
             rem no-op
         )
     )
 )
 
-set MILL=%MILL_DOWNLOAD_PATH%\!FULL_MILL_VERSION!!MILL_EXT!
+set MILL=%MILL_FINAL_DOWNLOAD_FOLDER%\!MILL_FULL_VERSION!!MILL_DOWNLOAD_EXT!
 
 set MILL_RESOLVE_DOWNLOAD=
 
@@ -289,34 +236,26 @@ if [!MILL_RESOLVE_DOWNLOAD!]==[true] (
     )
 
     rem there seems to be no way to generate a unique temporary file path (on native Windows)
-    set MILL_DOWNLOAD_FILE=%MILL%.tmp
+    if defined MILL_OUTPUT_DIR (
+        set MILL_TEMP_DOWNLOAD_FILE=%MILL_OUTPUT_DIR%\mill-temp-download
+        if not exist "%MILL_OUTPUT_DIR%" mkdir "%MILL_OUTPUT_DIR%"
+    ) else (
+        set MILL_TEMP_DOWNLOAD_FILE=out\mill-bootstrap-download
+        if not exist "out" mkdir "out"
+    )
 
     echo Downloading mill !MILL_VERSION! from !MILL_DOWNLOAD_URL! ... 1>&2
 
-    if not exist "%MILL_DOWNLOAD_PATH%" mkdir "%MILL_DOWNLOAD_PATH%"
-    rem curl is bundled with recent Windows 10
-    rem but I don't think we can expect all the users to have it in 2019
-    where /Q curl
-    if !ERRORLEVEL! EQU 0 (
-        curl -f -L "!MILL_DOWNLOAD_URL!" -o "!MILL_DOWNLOAD_FILE!"
-    ) else (
-        rem bitsadmin seems to be available on Windows 7
-        rem without /dynamic, github returns 403
-        rem bitsadmin is sometimes needlessly slow but it looks better with /priority foreground
-        bitsadmin /transfer millDownloadJob /dynamic /priority foreground "!MILL_DOWNLOAD_URL!" "!MILL_DOWNLOAD_FILE!"
-    )
-    if not exist "!MILL_DOWNLOAD_FILE!" (
-        echo Could not download mill !MILL_VERSION! 1>&2
-        exit /b 1
-    )
+    curl -f -L "!MILL_DOWNLOAD_URL!" -o "!MILL_TEMP_DOWNLOAD_FILE!"
 
-    move /y "!MILL_DOWNLOAD_FILE!" "%MILL%"
+    if not exist "%MILL_FINAL_DOWNLOAD_FOLDER%" mkdir "%MILL_FINAL_DOWNLOAD_FOLDER%"
+    move /y "!MILL_TEMP_DOWNLOAD_FILE!" "%MILL%"
 
-    set MILL_DOWNLOAD_FILE=
+    set MILL_TEMP_DOWNLOAD_FILE=
     set MILL_DOWNLOAD_SUFFIX=
 )
 
-set MILL_DOWNLOAD_PATH=
+set MILL_FINAL_DOWNLOAD_FOLDER=
 set MILL_VERSION=
 set MILL_REPO_URL=