chore(ci): build channel from artifacts (#395)

* chore: bundle artifacts as channel

* Include pixi-build-api version

* Fix ros backend

* Revert activation script

* Try ninja

* Fix regression

* Fix comments

* Bump rust compiler version

* Fix other comment

* Typo

* Use sccache for ubuntu

* Update sccache

* Fix sccache invocation

* Use --no-build-id to make sccache work

* Remove task caching

* Rework

* Set CARGO_HOME

* Add task caching

* Fix windows

---------

Co-authored-by: Julian Hofer <[email protected]>
Co-authored-by: Hofer-Julian <[email protected]>

Author: remimimimimi

.github/workflows/testsuite.yml

@@ -11,44 +11,46 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  RUST_LOG: info
-  RUST_BACKTRACE: 1
-  RUSTFLAGS: "-D warnings"
-  CARGO_TERM_COLOR: always
-  TARGET_DIR: target/pixi/ci
   PYTHONIOENCODING: utf-8
 
 jobs:
   build-binary-linux-x86_64:
     name: Build Ubuntu
     runs-on: ubuntu-latest
     timeout-minutes: 20
+    env:
+      SCCACHE_GHA_ENABLED: "true"
+      ACTIONS_CACHE_SERVICE_V2: on
     steps:
       - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
       - uses: prefix-dev/setup-pixi@main
         with:
           cache-write: ${{ github.event_name == 'push' && github.ref_name == 'main' }}
-      - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2
+      - name: Configure sccache
+        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
         with:
-          workspaces: ". -> target/pixi"
-          key: ${{ hashFiles('pixi.lock') }}
-          save-if: ${{ github.ref == 'refs/heads/main' }}
-      - name: Build backends
-        run: pixi run build-ci
+          script: |
+            core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || '');
+            core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
+      - name: Build backends channel
+        run: pixi run -e build create-testsuite-channel
+      - name: Show sccache stats
+        run: pixi run -e build sccache --stop-server
       - name: Upload executables as artifacts
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
           name: pixi-build-backends-linux-x86_64
-          path: |
-            ${{ env.TARGET_DIR }}/pixi-build-*
-            !${{ env.TARGET_DIR }}/pixi-build-*.d
+          path: artifacts-channel
           if-no-files-found: error
           retention-days: 60
 
   build-binary-windows-x86_64:
     name: Build Windows
     runs-on: windows-latest
     timeout-minutes: 20
+    env:
+      SCCACHE_GHA_ENABLED: "true"
+      ACTIONS_CACHE_SERVICE_V2: on
     steps:
       - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
       - name: Create Dev Drive
@@ -60,47 +62,55 @@ jobs:
         with:
           cache-write: ${{ github.event_name == 'push' && github.ref_name == 'main' }}
           manifest-path: ${{ env.PIXI_WORKSPACE }}/pixi.toml
-      - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2
+      - name: Configure sccache
+        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
         with:
-          workspaces: ". -> ${{ env.PIXI_WORKSPACE }}/target/pixi"
-          key: ${{ hashFiles('pixi.lock') }}
-          save-if: ${{ github.ref == 'refs/heads/main' }}
-      - name: Build backends
-        run: pixi run build-ci
+          script: |
+            core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || '');
+            core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
+      - name: Enable long paths (Windows)
+        run: git config --global core.longpaths true
+      - name: Build backends channel
+        run: pixi run -e build create-testsuite-channel
+        working-directory: ${{ env.PIXI_WORKSPACE }}
+      - name: Show sccache stats
+        run: pixi run -e build sccache --stop-server
         working-directory: ${{ env.PIXI_WORKSPACE }}
       - name: Upload executables as artifacts
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
           name: pixi-build-backends-windows-x86_64
-          path: |
-            ${{ env.PIXI_WORKSPACE }}/${{ env.TARGET_DIR }}/pixi-build-*
-            !${{ env.PIXI_WORKSPACE }}/${{ env.TARGET_DIR }}/pixi-build-*.d
+          path: ${{ env.PIXI_WORKSPACE }}/artifacts-channel
           if-no-files-found: error
           retention-days: 60
 
   build-binary-macos-aarch64:
     name: Build macOS
     runs-on: macos-latest
     timeout-minutes: 20
+    env:
+      SCCACHE_GHA_ENABLED: "true"
+      ACTIONS_CACHE_SERVICE_V2: on
     steps:
       - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
       - uses: prefix-dev/setup-pixi@main
         with:
           cache-write: ${{ github.event_name == 'push' && github.ref_name == 'main' }}
-      - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2
+      - name: Configure sccache
+        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
         with:
-          workspaces: ". -> target/pixi"
-          key: ${{ hashFiles('pixi.lock') }}
-          save-if: ${{ github.ref == 'refs/heads/main' }}
-      - name: Build backends
-        run: pixi run build-ci
+          script: |
+            core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || '');
+            core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
+      - name: Build backends channel
+        run: pixi run -e build create-testsuite-channel
+      - name: Show sccache stats
+        run: pixi run -e build sccache --stop-server
       - name: Upload executables as artifacts
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
           name: pixi-build-backends-macos-aarch64
-          path: |
-            ${{ env.TARGET_DIR }}/pixi-build-*
-            !${{ env.TARGET_DIR }}/pixi-build-*.d
+          path: artifacts-channel
           if-no-files-found: error
           retention-days: 60
 
@@ -135,9 +145,6 @@ jobs:
 
       - name: Run integration tests
         run: pixi run --locked test-slow
-        env:
-          PIXI_BIN_DIR: ${{ github.workspace }}/artifacts
-          BUILD_BACKENDS_BIN_DIR: ${{ github.workspace }}/artifacts
 
   test-windows-x86_64:
     name: Test Windows x86_64
@@ -177,9 +184,6 @@ jobs:
       - name: Run integration tests
         run: pixi run --locked test-slow
         working-directory: ${{ env.PIXI_WORKSPACE }}
-        env:
-          PIXI_BIN_DIR: ${{ env.PIXI_WORKSPACE }}/artifacts
-          BUILD_BACKENDS_BIN_DIR: ${{ env.PIXI_WORKSPACE }}/artifacts
 
   test-macos-aarch64:
     name: Test macOS aarch64
@@ -212,6 +216,3 @@ jobs:
 
       - name: Run integration tests
         run: pixi run --locked test-slow
-        env:
-          PIXI_BIN_DIR: ${{ github.workspace }}/artifacts
-          BUILD_BACKENDS_BIN_DIR: ${{ github.workspace }}/artifacts

.gitignore

@@ -47,3 +47,5 @@ AGENTS.md
 # Python
 *.pyc
 __pycache__/
+
+artifacts-channel

backends/pixi-build-ros/templates/bld_ament_cmake.bat

@@ -18,11 +18,11 @@ set CXX=cl.exe
 if not exist "%BUILD_DIR%" mkdir "%BUILD_DIR%"
 pushd %BUILD_DIR%
 
-:: set "CMAKE_GENERATOR=Ninja"
+set "CMAKE_GENERATOR=Ninja"
 :: We use the Visual Studio generator as a workaround for
 :: problems in Ninja when using long paths, see https://github.com/RoboStack/ros-humble/pull/229#issuecomment-2564856467
 :: Once those are solved, we can switch back to use Ninja
-set "CMAKE_GENERATOR=Visual Studio %VS_MAJOR% %VS_YEAR%"
+:: set "CMAKE_GENERATOR=Visual Studio %VS_MAJOR% %VS_YEAR%"
 
 :: PYTHON_INSTALL_DIR should be a relative path, see
 :: https://github.com/ament/ament_cmake/blob/2.3.2/ament_cmake_python/README.md

backends/pixi-build-ros/templates/build_ament_python.sh

@@ -16,7 +16,7 @@ if [ -f setup.cfg ] && grep -q "install[-_]scripts" setup.cfg; then
   PKG_NAME_SHORT=${PKG_NAME_SHORT//-/_}
   INSTALL_SCRIPTS_ARG="--install-scripts=$PREFIX/lib/$PKG_NAME_SHORT"
   echo "WARNING: setup.cfg not set, will set INSTALL_SCRIPTS_ARG to: $INSTALL_SCRIPTS_ARG"
-  $PYTHON setup.py install --prefix="$PREFIX" --install-lib="$SP_DIR" $INSTALL_SCRIPTS_ARG --single-version-externally-managed
+  $PYTHON setup.py install --prefix="$PREFIX" --install-lib="$SP_DIR" $INSTALL_SCRIPTS_ARG --single-version-externally-managed --record=files.txt
   # Remove build artifacts from setup.py install
   rm -rf *.egg-info 2>/dev/null || true
   rm -rf build/ 2>/dev/null || true

crates/pixi-build-backend/src/traits/project.rs

@@ -25,7 +25,7 @@ pub trait ProjectModel {
     fn dependencies(
         &self,
         platform: Option<Platform>,
-    ) -> Dependencies<<<Self as ProjectModel>::Targets as Targets>::Spec> {
+    ) -> Dependencies<'_, <<Self as ProjectModel>::Targets as Targets>::Spec> {
         self.targets()
             .map(|t| t.dependencies(platform))
             .unwrap_or_default()

crates/pixi-build-backend/src/traits/targets.rs

@@ -81,7 +81,7 @@ pub trait Targets {
     fn targets(&self) -> impl Iterator<Item = (&Self::Selector, &Self::Target)>;
 
     /// Return all dependencies for the given platform
-    fn dependencies(&self, platform: Option<Platform>) -> Dependencies<Self::Spec>;
+    fn dependencies(&self, platform: Option<Platform>) -> Dependencies<'_, Self::Spec>;
 
     /// Return the run dependencies for the given platform
     fn run_dependencies(
@@ -191,7 +191,7 @@ impl Targets for pbt::TargetsV1 {
             .collect::<IndexMap<&pbt::SourcePackageName, &pbt::PackageSpecV1>>()
     }
 
-    fn dependencies(&self, platform: Option<Platform>) -> Dependencies<Self::Spec> {
+    fn dependencies(&self, platform: Option<Platform>) -> Dependencies<'_, Self::Spec> {
         let build_deps = self.build_dependencies(platform);
         let host_deps = self.host_dependencies(platform);
         let run_deps = self.run_dependencies(platform);

crates/recipe-stage0/src/recipe.rs

@@ -293,7 +293,7 @@ impl<T> ListOrItem<T> {
         self.0.len()
     }
 
-    pub fn iter(&self) -> std::slice::Iter<T> {
+    pub fn iter(&self) -> std::slice::Iter<'_, T> {
         self.0.iter()
     }
 }
@@ -659,13 +659,13 @@ impl Display for ConditionalRequirements {
     }
 }
 
-#[derive(Debug, Serialize, Deserialize)]
-pub(crate) struct Requirements {
-    pub build: Vec<SerializableMatchSpec>,
-    pub host: Vec<SerializableMatchSpec>,
-    pub run: Vec<SerializableMatchSpec>,
-    pub run_constraints: Vec<SerializableMatchSpec>,
-}
+// #[derive(Debug, Serialize, Deserialize)]
+// pub(crate) struct Requirements {
+//     pub build: Vec<SerializableMatchSpec>,
+//     pub host: Vec<SerializableMatchSpec>,
+//     pub run: Vec<SerializableMatchSpec>,
+//     pub run_constraints: Vec<SerializableMatchSpec>,
+// }
 
 #[derive(Debug, Serialize, Deserialize, Clone, Default)]
 pub struct Test {