Removing compiler build and using rustup (#33)

Changing to use `rustup` for the Rust compiler now that we are not on
our own fork.

- [x] Need `build.rs` so that correct rust version is used with
components installed
- [x] Need to update README.md
- [x] Building with `Makefile` needs to be removed
- [x] Handling tests with `Makefile` needs to change to use `cargo`
- [x] CI needs to be updated succesfully
- [x] Need to handle golden tests difference
- [x] Need to make sure this will still work with `mir-semantics` ([see
this PR](runtimeverification/mir-semantics#439))

Moved to future work:
- [ ] Change tests to use `cargo test` ? Could remove `Makefile`
entirely then
- [ ] Need to make sure this will still work with extracting
stable-mir-json for cargo projects / `std lib`

Author: dkcumming

.github/workflows/test.yml

@@ -28,33 +28,12 @@ jobs:
       - name: "Set up nightly Rust" # https://github.com/rust-lang/rustup/issues/3409
         uses: dtolnay/rust-toolchain@master
         with:
-          toolchain: nightly-2024-08-28
+          toolchain: nightly-2024-11-29 # Hardcoded version, same as is in the build.rs
 
-      - name: 'Set up tree for rust dependency'
-        run: make setup
-
-      - name: 'Cache smir_pretty and rustc'
-        uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: |
-            .
-            deps/rust/src
-          cache-directories: |
-            target
-            deps/rust/src/build
-            deps/rust/src/target
-
-      - name: 'Build smir_pretty and its rustc dependency'
-        run: | # rustc bootstrap checks this and refuses stage 1 in "CI"
-          export GITHUB_ACTIONS="in denial" && \
-          echo "GITHUB_ACTIONS = ${GITHUB_ACTIONS}" && \
-          make build_all
-
-      - name: 'Run smir integration tests'
+      - name: 'Build smir_pretty'
         run: |
-          make integration-test
+          cargo build -vv
 
-      - name: 'Clean up toolchain'
-        if: always()
+      - name: 'Run smir integration tests'
         run: |
-          make rustup-clear-toolchain
+          make integration-test

Cargo.lock

@@ -2,18 +2,13 @@
 name = "smir_pretty"
 version = "0.1.0"
 edition = "2021"
-# rust-version = "1.78.0" # I think we get latest available rust by unsetting this and setting rust-toolchain.toml.toolchain.channel = nightly
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
 dot-writer = "0.1.4"
 tracing = "0.1"
-# serde = { version = "=1.0.202", features = ["derive"] }
-# serde_cbor = "0.11"
-# serde_json = "1.0"
-# tar = "0.4"
 
 [package.metadata.rust-analyzer]
 # This package uses rustc crates.
-rustc_private=true
+rustc_private=true

Cargo.toml

@@ -1,115 +1,30 @@
-TARGET ?= debug
-STAGE  ?= 1
-ifneq (0, $(shell test "${STAGE}" -gt 0 2>/dev/null; echo "$$?"))
-$(error STAGE must be set to a number greater than 0)
-endif
-ifneq (${TARGET}, $(filter ${TARGET},debug release))
-$(error TARGET must be set to one of debug/release)
-endif
-RUST_DIR=${CURDIR}/deps/rust
-STAGE_FILE=${RUST_DIR}/stage
-RUST_SRC=${RUST_DIR}/src
-RUST_ARCH=$(shell "${PWD}"/rustc_arch.sh)
-RUST_BUILD_DIR=${RUST_SRC}/build/${RUST_ARCH}
-RUST_INSTALL_DIR=${RUST_BUILD_DIR}/stage${STAGE}
-RUST_LIB_DIR=${RUST_INSTALL_DIR}/lib
-RUST_DEP_DIR=${RUST_BUILD_DIR}/stage1-rustc/${RUST_ARCH}/release/deps
-TARGET_DEP_DIR=${CURDIR}/target/${TARGET}/deps
-TEMP_DIR=${RUST_DIR}/temp
-#############################################
-# depend on the rust compiler
-RUST_REPO=https://github.com/rust-lang/rust
-# tip of the `beta` branch on 2025-01-14
-RUST_BRANCH=beta
-RUST_COMMIT=fe9b975
-#############################################
-TOOLCHAIN_NAME=smir_pretty
 RELEASE_FLAG=
-ifeq (${TARGET}, release)
-RELEASE_FLAG=--release
-endif
+TOOLCHAIN_NAME=''
 
 default: build
 
-build_all: rust_build rust_set_toolchain build
-
-setup: rust_clone
-
-update: ${RUST_SRC}
-	cd "${RUST_SRC}"; git fetch origin; git checkout ${RUST_COMMIT}
-
 build:
 	cargo build ${RELEASE_FLAG}
 
-clean:
-	cd "${RUST_SRC}" && ./x.py clean
-	-rm -r "${TEMP_DIR}"
-	-rm -r "${RUST_DIR}"/tests
-	-rm -r ./target
-
-distclean:
-	cd "${RUST_SRC}" && git clean -dffx
-	-rm -r "${TEMP_DIR}"
-	-rm -r "${RUST_DIR}"/tests
-	-rm -r ./target
-
-# this clean removes old backup files which accumulate and lead to slow build times
-prebuild_clean: ${RUST_SRC}
-	-find -name '*.old' -delete
-	-rm -r "${TEMP_DIR}"
-
-# NOTE: a deeper clone depth is needed for the build process
-rust_clone:
-	git clone --depth 70 --single-branch --branch "${RUST_BRANCH}" "${RUST_REPO}" "${RUST_SRC}" && \
-	cd "${RUST_SRC}" && \
-	git checkout ${RUST_COMMIT}
-
-
-# rust_build for linking against custom rustc is involved
-#
-# 1. core rust compiler must be built via ./x.py build/install (we also build the test harness here)
-# 2. rustc-dev component must be installed (./x.py build/install does _not_ handle, must be done manually)
-# 3. HACK(only for ./x.py install) we copy required libraries to the libdir
-# 4. finally, use rustup to create custom toolchain
-
-rust_build: ${RUST_SRC} prebuild_clean
-	cd "${RUST_SRC}"; ./x.py build src/tools/compiletest
-	cd "${RUST_SRC}"; ./x.py build --stage ${STAGE} --set rust.debug-logging=true compiler/rustc library/std
-	cd "${RUST_SRC}"; ./x.py dist --set rust.debug-logging=true rustc-dev
-	mkdir -p "${TEMP_DIR}"
-	cd "${RUST_SRC}"; tar xf ./build/dist/rustc-dev*tar.gz -C "${TEMP_DIR}"
-	"${TEMP_DIR}"/rustc-dev*/install.sh --prefix="${RUST_INSTALL_DIR}" --sysconfdir="${RUST_INSTALL_DIR}" > "${RUST_DIR}"/rustc-dev-install.log 2>&1
-
-rust_lib_copy:
-	cd "${RUST_LIB_DIR}"; cp libLLVM* rustlib/*/lib/
-
-rust_set_toolchain: ${RUST_LIB_DIR}
-	rustup toolchain link "${TOOLCHAIN_NAME}" "${RUST_INSTALL_DIR}"
-	rustup override set "${TOOLCHAIN_NAME}"
-	echo ${STAGE} > ${STAGE_FILE}
+clean: rustup-clear-toolchain
+	cargo clean
 
 .PHONY: rustup-clear-toolchain
 rustup-clear-toolchain:
 	rustup override unset
 	rustup override unset --nonexistent
 	rustup toolchain uninstall "${TOOLCHAIN_NAME}"
 
-generate_ui_tests:
-	mkdir -p "${RUST_DIR}"/tests
-	cd "${RUST_SRC}"; ./get_runpass.sh tests/ui > "${RUST_DIR}"/tests_ui_sources
-	-cd "${RUST_SRC}"; ./ui_compiletest.sh "${RUST_SRC}" "${RUST_DIR}"/tests/ui/upstream "${RUST_DIR}"/tests_ui_sources --pass check --force-rerun 2>&1 > "${RUST_DIR}"/tests_ui_upstream.log
-	-cd "${RUST_SRC}"; RUST_BIN="${PWD}"/run.sh ./ui_compiletest.sh "${RUST_SRC}" "${RUST_DIR}"/tests/ui/smir "${RUST_DIR}"/tests_ui_sources --pass check --force-rerun 2>&1 > "${RUST_DIR}"/tests_ui_smir.log
-
 TESTDIR=$(CURDIR)/tests/integration/programs
 
 .PHONY: integration-test
 integration-test: TESTS     ?= $(shell find $(TESTDIR) -type f -name "*.rs")
-integration-test: SMIR      ?= $(CURDIR)/run.sh -Z no-codegen
+integration-test: SMIR      ?= cargo run -- "-Zno-codegen"
 # override this to tweak how expectations are formatted
 integration-test: NORMALIZE ?= jq -S -e -f $(TESTDIR)/../normalise-filter.jq
 # override this to re-make golden files
 integration-test: DIFF      ?= | diff -
-integration-test: build
+integration-test:
 	errors=""; \
 	report() { echo "$$1: $$2"; errors="$$errors\n$$1: $$2"; }; \
 	for rust in ${TESTS}; do \

Makefile

@@ -1,33 +1,16 @@
 # Rust Stable MIR Pretty Printing
 
-This package provides:
-
-1.  a library crate that provides:
-    -   a `rustc` compiler wrapper which can access stable MIR APIs
-    -   a pretty-printer for a large fragment of stable MIR
-2.  a `rustc` wrapper binary that uses (1)-(2) to pretty-print Rust source files as stable MIR using the `.smir.json` extension.
-
-It is designed so that anyone can use this library crate as a jumping off point for their own tools which might use stable MIR APIs.
+This package provides a program that will emit a JSON serialisation of the Stable MIR of a Rust program
 
 ## Building
 
-For first-time builds, run:
-
-```shell
-make setup build_all
-```
-
-If the underlying `rustc` branch is updated and this crate needs to be rebuilt on top of it, run:
-
 ```shell
-make update build_all
+cargo build
 ```
 
-If the source code changes locally for this crate only and it needs to be rebuilt, run:
+NOTE: requries [rustup](https://www.rust-lang.org/tools/install)
 
-```shell
-make build
-```
+The `build.rs` script will ensure that the correct version of rust and the required components are installed and defaulted. What `rustup` commands are run can be seen by adding verbosity flag `-vv` to `cargo`.
 
 ## Usage
 
@@ -36,7 +19,7 @@ The options that this tool accepts are identical to `rustc`.
 To generate stable MIR output without building a binary, you can invoke the tool as follows:
 
 ```shell
-./run.sh -Z no-codegen <crate_root>
+cargo run -- <rustc_flags> <path_from_crate_root>
 ```
 
 There is experimental support for rendering the Stable-MIR items and their basic blocks as a 
@@ -53,17 +36,6 @@ There are a few environment variables that can be set to control the tools outpu
 2.  `LINK_INST`  - use a richer key-structure for the link-time `functions` map which uses keys that are pairs of a function type (`Ty`) _and_ an function instance kind (`InstanceKind`)
 3.  `DEBUG` - serialize additional data in the JSON file and dump logs to stdout
 
-### Invocation Details
-
-We use an uncommon build process where we link against a patched rustc installed in this repo.
-However, since `cargo build` does not set `rpath` for dynamic linking, we must manually point the program loader/dynamic linker at the required runtime libraries.
-Note that the `cargo run` command appears to prepard the rustlib directories automatically to the dynamic link search path.
-If you wish to run the tool manually, you will need to tell the program loader/dynamic linker where to find the missing libraries by:
-
-1.  setting `LD_LIBRARY_PATH`
-2.  setting the `rpath` attribute on the binary ELF file
-3.  manually invoking the loader (usually `/usr/lib/ld-linux.so.2`) with its specific options
-
 ## Tests
 
 ### Running the Tests

README.md

@@ -0,0 +1,31 @@
+use std::process::Command;
+
+fn main() {
+    let status = Command::new("rustup")
+        .args(&["install", "nightly-2024-11-29"])
+        .status()
+        .expect("build.rs failed to install nightly-2024-11-29");
+
+    println!("Installed nightly-2024-11-29: {}", status);
+
+    let status = Command::new("rustup")
+        .args(&["default", "nightly-2024-11-29"])
+        .status()
+        .expect("build.rs failed to default nightly-2024-11-29");
+
+    println!("Defaulted nightly-2024-11-29: {}", status);
+
+    let status = Command::new("rustup")
+        .args(&["component", "add", "rustc-dev"])
+        .status()
+        .expect("build.rs failed to install rustc-dev");
+
+    println!("Added component rustc-dev: {}", status);
+
+    let status = Command::new("rustup")
+        .args(&["component", "add", "llvm-tools"])
+        .status()
+        .expect("build.rs failed to install llvm-tools");
+
+    println!("Added component llvm-tools: {}", status);
+}