Merge pull request #229 from traversaro/rebuild-christmas-2024-linux-64-rattler-build

Rebuild January 2025 - switch to rattler-build

Author: Tobias-Fischer

.ci_support/conda_forge_pinnings.yaml

@@ -0,0 +1,2 @@
+# GitHub syntax highlighting
+pixi.lock linguist-language=YAML linguist-generated=true

.gitattributes

@@ -1,3 +1,5 @@
+name: Generate CI pipelines for ROS 2 packages
+
 permissions:
   actions: write
   contents: write
@@ -18,7 +20,7 @@ jobs:
         python-version: '3.11' # Version range or exact version of a Python version to use, using SemVer's version range syntax
     - name: Install vinca
       run: |
-        pip install git+https://github.com/RoboStack/vinca.git
+        pip install git+https://github.com/RoboStack/vinca.git@rattler-build-humble
 
     - name: Generate recipes for linux-64
       run: |

.github/testpr_environment.yml

@@ -8,71 +8,51 @@ env:
 
 jobs:
   build:
-    runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest, macos-14, windows-2019]
-        platform: [linux-64, linux-aarch64, osx-64, win-64, osx-arm64]
-        exclude:
-          - os: ubuntu-latest
-            platform: win-64
-          - os: ubuntu-latest
-            platform: osx-64
+        include:
           - os: ubuntu-latest
-            platform: osx-arm64
-          - os: macos-14
             platform: linux-64
-          - os: macos-14
+          - os: cirun-linux-aarch64--${{ github.run_id }}
             platform: linux-aarch64
-          - os: macos-14
+          - os: macos-13
             platform: osx-64
           - os: macos-14
-            platform: win-64
-          - os: macos-latest
-            platform: linux-64
-          - os: macos-latest
-            platform: linux-aarch64
-          - os: macos-latest
             platform: osx-arm64
-          - os: macos-latest
-            platform: win-64
           - os: windows-2019
-            platform: linux-64
-          - os: windows-2019
-            platform: linux-aarch64
-          - os: windows-2019
-            platform: osx-64
-          - os: windows-2019
-            platform: osx-arm64
+            platform: win-64
+
+    runs-on: ${{ matrix.os }}
+
     steps:
+    - uses: actions/checkout@v4
+      with:
+        persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal token
+        fetch-depth: 0 # otherwise, you will failed to push refs to dest repo
+
+    - uses: prefix-dev/[email protected]
+      with:
+        environments: beta
+        frozen: true
+
     # Workaround for https://github.com/RoboStack/ros-humble/pull/141#issuecomment-1941919816
     - name: Clean up PATH
       if: contains(matrix.os, 'windows')
       uses: egor-tensin/cleanup-path@v4
       with:
         # cygpath in C:\Program Files\Git\usr\bin is used by install micromamba
-        # git in C:\Program Files\Git\bin is used by pip install git+ 
+        # git in C:\Program Files\Git\bin is used by pip install git+
         dirs: 'C:\Program Files\Git\usr\bin;C:\Program Files\Git\bin;C:\Program Files\Git\cmd;C:\Program Files\Git\mingw64\bin'
 
-    - uses: actions/checkout@master
-    - name: install micromamba
-      uses: mamba-org/setup-micromamba@main
-      with:
-        environment-file: .github/testpr_environment.yml
-        condarc: |
-          channels:
-            - robostack-staging
-            - conda-forge
-
-    - shell: bash -l {0}
-      if: matrix.platform == 'linux-aarch64'
-      run: |
-        echo "::group::Configure binfmt_misc"
-        docker run --rm --privileged multiarch/qemu-user-static:register --reset --credential yes
-        export FEEDSTOCK_NAME="$(basename $GITHUB_REPOSITORY)"
-        export GIT_BRANCH="$(basename $GITHUB_REF)"
-        echo "::endgroup::"
+    # - shell: bash -l {0}
+    #   if: matrix.platform == 'linux-aarch64'
+    #   run: |
+    #     echo "::group::Configure binfmt_misc"
+    #     docker run --rm --privileged multiarch/qemu-user-static:register --reset --credential yes
+    #     export FEEDSTOCK_NAME="$(basename $GITHUB_REPOSITORY)"
+    #     export GIT_BRANCH="$(basename $GITHUB_REF)"
+    #     echo "::endgroup::"
 
     - name: Install libgl1-mesa-dev (only for linux-aarch64)
       if: matrix.platform == 'linux-aarch64'
@@ -81,24 +61,15 @@ jobs:
         sudo apt-get update
         sudo apt-get install -y libgl1-mesa-dev libglu1-mesa-dev libx11-dev libxi-dev libxmu-dev
 
-    - shell: bash -l {0}
-      run: |
-        conda config --show channels | grep defaults && conda config --remove channels defaults
-        conda info
-        conda config --show
-        micromamba install -y pip rospkg networkx "catkin_pkg>=0.4.16" "ruamel.yaml>=0.16.6" "rosdistro>=0.8.0" "empy>=3.3.4" boa
-    - name: install vinca
-      shell: bash -l {0}
-      run: |
-        # use no-deps for now, otherwise problems with ruamel.
-        pip install git+https://github.com/RoboStack/vinca.git --no-deps
     # For some reason, the Strawberry perl's pkg-config is found
     # instead of the conda's one, so let's delete the /c/Strawberry directory
+    # Furthermore, we also need to remove an older SDK that is used and can result in compilation problems
     - name: Debug pkg-config problem
       if: contains(matrix.os, 'windows')
       shell: bash -l {0}
       run: |
         rm -rf /c/Strawberry
+        rm -rf "/c/Program Files (x86)/Windows Kits/10/Include/10.0.17763.0/"
 
     - name: Check what files have changed
       id: filecheck
@@ -119,85 +90,82 @@ jobs:
         echo "::set-output name=WIN_YAML_CHANGED::${?}"
     - name: Generate recipes for linux-64
       shell: bash -l {0}
-      if: steps.filecheck.outputs.LINUX_YAML_CHANGED == 1 && matrix.os == 'ubuntu-latest' && matrix.platform == 'linux-64'
+      if: steps.filecheck.outputs.LINUX_YAML_CHANGED == 1 && matrix.platform == 'linux-64'
       run: |
-        git clean -fdx
         cp vinca_linux_64.yaml vinca.yaml
-        vinca --platform linux-64
-        ls
-        cat recipe.yaml
+        mkdir -p recipes
+        $HOME/.pixi/bin/pixi run -e beta -v vinca --platform linux-64 -m -n
+        ls -la recipes
     - name: Generate recipes for linux-aarch64
       shell: bash -l {0}
-      if: steps.filecheck.outputs.LINUX_AARCH_YAML_CHANGED == 1 && matrix.os == 'ubuntu-latest' && matrix.platform == 'linux-aarch64'
+      if: steps.filecheck.outputs.LINUX_AARCH_YAML_CHANGED == 1 && matrix.platform == 'linux-aarch64'
       run: |
-        git clean -fdx
         cp vinca_linux_aarch64.yaml vinca.yaml
-        vinca --platform linux-aarch64
-        ls
-        cat recipe.yaml
+        mkdir -p recipes
+        $HOME/.pixi/bin/pixi run -e beta -v vinca --platform linux-aarch64 -m -n
+        ls -la recipes
     - name: Generate recipes for osx-64
       shell: bash -l {0}
-      if: steps.filecheck.outputs.OSX_YAML_CHANGED == 1 && matrix.os == 'macos-latest' && matrix.platform == 'osx-64'
+      if: steps.filecheck.outputs.OSX_YAML_CHANGED == 1 && matrix.platform == 'osx-64'
       run: |
-        git clean -fdx
         cp vinca_osx.yaml vinca.yaml
-        vinca --platform osx-64
-        ls
-        cat recipe.yaml
+        mkdir -p recipes
+        $HOME/.pixi/bin/pixi run -e beta -v vinca --platform osx-64 -m -n
+        ls -la recipes
     - name: Generate recipes for osx-arm64
       shell: bash -l {0}
-      if: steps.filecheck.outputs.OSX_ARM_YAML_CHANGED == 1 && matrix.os == 'macos-14' && matrix.platform == 'osx-arm64'
+      if: steps.filecheck.outputs.OSX_ARM_YAML_CHANGED == 1 && matrix.platform == 'osx-arm64'
       run: |
-        git clean -fdx
         cp vinca_osx_arm64.yaml vinca.yaml
-        vinca --platform osx-arm64
-        ls
-        cat recipe.yaml
+        mkdir -p recipes
+        $HOME/.pixi/bin/pixi run -e beta -v vinca --platform osx-arm64 -m -n
+        ls -la recipes
     - name: Generate recipes for win-64
       shell: bash -l {0}
-      if: steps.filecheck.outputs.WIN_YAML_CHANGED == 1 && matrix.os == 'windows-2019' && matrix.platform == 'win-64'
+      if: steps.filecheck.outputs.WIN_YAML_CHANGED == 1 && matrix.platform == 'win-64'
       run: |
+        # Workaround for problem related to long paths
         echo "CONDA_BLD_PATH=C:\\bld\\" >> $GITHUB_ENV
-        git clean -fdx
+        mkdir /c/bld
         cp vinca_win.yaml vinca.yaml
-        vinca --platform win-64
-        ls
-        cat recipe.yaml
+        mkdir -p recipes
+        $HOME/.pixi/bin/pixi run -e beta -v vinca --platform win-64 -m -n
+        ls -la recipes
     - name: Check if there are packages to be built
       id: newrecipecheck
       shell: bash -l {0}
       run: |
         # continue on error
         set +e
-        grep -q "outputs: \[\]" recipe.yaml > /dev/null
+        test ! -d recipes
         echo "::set-output name=RECIPE_CREATED::${?}"
     - name: Build recipes for linux-64
       shell: bash -l {0}
+      if: steps.newrecipecheck.outputs.RECIPE_CREATED == 1 && steps.filecheck.outputs.LINUX_YAML_CHANGED == 1 && matrix.platform == 'linux-64'
       run: |
-        boa build additional_recipes/ros2-distro-mutex/ -m ./.ci_support/conda_forge_pinnings.yaml -m ./conda_build_config.yaml
-        boa build . -m ./.ci_support/conda_forge_pinnings.yaml -m ./conda_build_config.yaml
-      if: steps.newrecipecheck.outputs.RECIPE_CREATED == 1 && steps.filecheck.outputs.LINUX_YAML_CHANGED == 1 && matrix.os == 'ubuntu-latest' && matrix.platform == 'linux-64'
+        env -i $HOME/.pixi/bin/pixi run -e beta rattler-build build --recipe additional_recipes/ros2-distro-mutex/ --target-platform linux-64 -m ./conda_build_config.yaml -c robostack-staging -c conda-forge --skip-existing
+        env -i $HOME/.pixi/bin/pixi run -e beta rattler-build build --recipe-dir recipes --target-platform linux-64 -m ./conda_build_config.yaml -c robostack-staging -c conda-forge --skip-existing
     - name: Build recipes for linux-aarch64
       shell: bash -l {0}
-      if: steps.newrecipecheck.outputs.RECIPE_CREATED == 1 && steps.filecheck.outputs.LINUX_AARCH_YAML_CHANGED == 1 && matrix.os == 'ubuntu-latest' && matrix.platform == 'linux-aarch64'
+      if: steps.newrecipecheck.outputs.RECIPE_CREATED == 1 && steps.filecheck.outputs.LINUX_AARCH_YAML_CHANGED == 1 && matrix.platform == 'linux-aarch64'
       run: |
-        boa build additional_recipes/ros2-distro-mutex/ -m ./.ci_support/conda_forge_pinnings.yaml -m ./conda_build_config.yaml --target-platform=linux-aarch64
-        boa build . -m ./.ci_support/conda_forge_pinnings.yaml -m ./conda_build_config.yaml --target-platform=linux-aarch64
+        env -i $HOME/.pixi/bin/pixi run -e beta rattler-build build --recipe additional_recipes/ros2-distro-mutex/ --target-platform linux-aarch64 -m ./conda_build_config.yaml -c robostack-staging -c conda-forge --skip-existing
+        env -i $HOME/.pixi/bin/pixi run -e beta rattler-build build --recipe-dir recipes --target-platform linux-aarch64 -m ./conda_build_config.yaml -c robostack-staging -c conda-forge --skip-existing
     - name: Build recipes for osx-64
       shell: bash -l {0}
-      if: steps.newrecipecheck.outputs.RECIPE_CREATED == 1 && steps.filecheck.outputs.OSX_YAML_CHANGED == 1 && matrix.os == 'macos-latest' && matrix.platform == 'osx-64'
+      if: steps.newrecipecheck.outputs.RECIPE_CREATED == 1 && steps.filecheck.outputs.OSX_YAML_CHANGED == 1 && matrix.platform == 'osx-64'
       run: |
-        boa build additional_recipes/ros2-distro-mutex/ -m ./.ci_support/conda_forge_pinnings.yaml -m ./conda_build_config.yaml
-        boa build . -m ./.ci_support/conda_forge_pinnings.yaml -m ./conda_build_config.yaml
+        env -i $HOME/.pixi/bin/pixi run -e beta rattler-build build --recipe additional_recipes/ros2-distro-mutex/ --target-platform osx-64 -m ./conda_build_config.yaml -c robostack-staging -c conda-forge --skip-existing
+        env -i $HOME/.pixi/bin/pixi run -e beta rattler-build build --recipe-dir recipes --target-platform osx-64 -m ./conda_build_config.yaml -c robostack-staging -c conda-forge --skip-existing
     - name: Build recipes for osx-arm64
       shell: bash -l {0}
-      if: steps.newrecipecheck.outputs.RECIPE_CREATED == 1 && steps.filecheck.outputs.OSX_ARM_YAML_CHANGED == 1 && matrix.os == 'macos-14' && matrix.platform == 'osx-arm64'
+      if: steps.newrecipecheck.outputs.RECIPE_CREATED == 1 && steps.filecheck.outputs.OSX_ARM_YAML_CHANGED == 1 && matrix.platform == 'osx-arm64'
       run: |
-        boa build additional_recipes/ros2-distro-mutex/ -m ./.ci_support/conda_forge_pinnings.yaml -m ./conda_build_config.yaml --target-platform=osx-arm64
-        boa build . -m ./.ci_support/conda_forge_pinnings.yaml -m ./conda_build_config.yaml
+        env -i $HOME/.pixi/bin/pixi run -e beta rattler-build build --recipe additional_recipes/ros2-distro-mutex/ --target-platform osx-arm64 -m ./conda_build_config.yaml -c robostack-staging -c conda-forge --skip-existing
+        env -i $HOME/.pixi/bin/pixi run -e beta rattler-build build --recipe-dir recipes --target-platform osx-arm64 -m ./conda_build_config.yaml -c robostack-staging -c conda-forge --skip-existing
     - name: Build recipes for win-64
       shell: bash -l {0}
-      if: steps.newrecipecheck.outputs.RECIPE_CREATED == 1 && steps.filecheck.outputs.WIN_YAML_CHANGED == 1 && matrix.os == 'windows-2019' && matrix.platform == 'win-64'
+      if: steps.newrecipecheck.outputs.RECIPE_CREATED == 1 && steps.filecheck.outputs.WIN_YAML_CHANGED == 1 && matrix.platform == 'win-64'
       run: |
-        boa build additional_recipes/ros2-distro-mutex/ -m ./.ci_support/conda_forge_pinnings.yaml -m ./conda_build_config.yaml
-        boa build . -m ./.ci_support/conda_forge_pinnings.yaml -m ./conda_build_config.yaml
+        $HOME/.pixi/bin/pixi run -e beta rattler-build build --recipe additional_recipes/ros2-distro-mutex/ --target-platform win-64 -m ./conda_build_config.yaml -c robostack-staging -c conda-forge --skip-existing
+        $HOME/.pixi/bin/pixi run -e beta rattler-build build --recipe-dir recipes -m ./conda_build_config.yaml --target-platform win-64 -c robostack-staging -c conda-forge --skip-existing

.github/workflows/main.yml

@@ -5,3 +5,7 @@ recipes/
 *.ps1
 *.json
 .DS_Store
+# pixi environments
+.pixi
+*.egg-info
+output/

.github/workflows/testpr.yml

@@ -9,67 +9,25 @@ set -xeuo pipefail
 export PYTHONUNBUFFERED=1
 export FEEDSTOCK_ROOT="${FEEDSTOCK_ROOT:-/home/conda/feedstock_root}"
 export RECIPE_ROOT="${RECIPE_ROOT:-/home/conda/recipe_root}"
-# export CI_SUPPORT="${FEEDSTOCK_ROOT}/.ci_support"
-# export CONFIG_FILE="${CI_SUPPORT}/${CONFIG}.yaml"
 
-cat >~/.condarc <<CONDARC
-conda-build:
-    root-dir: /opt/conda/build_artifacts
-CONDARC
+curl -fsSL https://pixi.sh/install.sh | bash
+export PATH="$HOME/.pixi/bin:$PATH"
 
-# This doesn't work, see e.g. https://dev.azure.com/robostack/ros_pipelines/_build/results?buildId=188&view=logs&j=4e20d398-0572-5e54-89c7-6bdb9c00a59a&t=f5885ff8-badf-54b3-1543-35164851bdf3
-# if grep -q libgl "recipes/${CURRENT_BUILD_PKG_NAME}.yaml"; then
-# 	sudo yum install -y install mesa-libGL-devel
-# fi
-
-export "CONDA_BLD_PATH=/opt/conda/build_artifacts"
-
-mkdir -p $CONDA_BLD_PATH
-conda index $CONDA_BLD_PATH
-
-conda config --set remote_max_retries 5
-conda config --add channels conda-forge
-conda config --add channels robostack-staging
-conda config --add channels $CONDA_BLD_PATH
-conda config --remove channels defaults
-# conda config --set channel_priority strict
-
-mamba update conda --yes --quiet -c conda-forge
-mamba install --yes --quiet pip conda-build anaconda-client mamba boa
-
-# setup_conda_rc "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}"
-# export PATH="$HOME/miniconda/bin:$PATH"
-conda config --set anaconda_upload yes
-conda config --set show_channel_urls true
-conda config --set auto_update_conda false
-conda config --set add_pip_as_python_dependency false
-
-conda info
-conda config --show-sources
-conda list --show-channel-urls
+export CONDA_BLD_PATH="/opt/conda/build_artifacts"
 
 pwd
 
-for recipe in ${CURRENT_RECIPES[@]}; do
-	cd ${FEEDSTOCK_ROOT}/recipes/${recipe}
-	boa build . -m ${FEEDSTOCK_ROOT}/.ci_support/conda_forge_pinnings.yaml -m ${FEEDSTOCK_ROOT}/conda_build_config.yaml
-done
-
-anaconda -t ${ANACONDA_API_TOKEN} upload /opt/conda/build_artifacts/linux-*/*.tar.bz2 --force
-# quetz-client "${QUETZ_URL}" /opt/conda/build_artifacts --force
-
-# set up the condarc
+cd ${FEEDSTOCK_ROOT}
 
-# source run_conda_forge_build_setup
-
-# # make the build number clobber
-# make_build_number "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}"
+for recipe in ${CURRENT_RECIPES[@]}; do
+	pixi run -v rattler-build build \
+		--recipe ${FEEDSTOCK_ROOT}/recipes/${recipe} \
+		-m ${FEEDSTOCK_ROOT}/conda_build_config.yaml \
+		-c robostack-staging -c conda-forge \
+		--output-dir $CONDA_BLD_PATH
 
-# conda build "${RECIPE_ROOT}" -m "${CI_SUPPORT}/${CONFIG}.yaml" \
-#     --clobber-file "${CI_SUPPORT}/clobber_${CONFIG}.yaml"
+		# -m ${FEEDSTOCK_ROOT}/.ci_support/conda_forge_pinnings.yaml \
 
-# if [[ "${UPLOAD_PACKAGES}" != "False" ]]; then
-#     upload_package  "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}"
-# fi
+done
 
-# touch "${FEEDSTOCK_ROOT}/build_artifacts/conda-forge-build-done-${CONFIG}"
+pixi run upload ${CONDA_BLD_PATH}/linux-*/*.conda --force