Merge pull request #395 from h-vetinari/artefact

Fix artefact persistence on windows

Author: h-vetinari

.github/workflows/conda-build.yml

@@ -79,6 +79,7 @@ jobs:
       uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
 
     - name: Build on Linux
+      id: build-linux
       if: matrix.os == 'ubuntu'
       env:
         CONFIG: ${{ matrix.CONFIG }}
@@ -109,6 +110,7 @@ jobs:
         ./.scripts/run_docker_build.sh
 
     - name: Build on macOS
+      id: build-macos
       if: matrix.os == 'macos'
       env:
         CONFIG: ${{ matrix.CONFIG }}
@@ -132,13 +134,19 @@ jobs:
         ./.scripts/run_osx_build.sh
 
     - name: Build on windows
+      id: build-windows
+      if: matrix.os == 'windows'
       shell: cmd
       run: |
         set "flow_run_id=github_%GITHUB_RUN_ID%"
         set "remote_url=https://github.com/%GITHUB_REPOSITORY%"
         set "sha=%GITHUB_SHA%"
         call ".scripts\run_win_build.bat"
       env:
+        # default value; make explicit for now (pending smithy-improvements),
+        # as it needs to be synchronized with artefact generation below; see
+        # https://github.com/conda-forge/conda-smithy/issues/2345
+        CONDA_BLD_DIR: C:\bld
         MINIFORGE_HOME: D:\Miniforge
         PYTHONUNBUFFERED: 1
         CONFIG: ${{ matrix.CONFIG }}
@@ -147,45 +155,72 @@ jobs:
         BINSTAR_TOKEN: ${{ secrets.BINSTAR_TOKEN }}
         FEEDSTOCK_TOKEN: ${{ secrets.FEEDSTOCK_TOKEN }}
         STAGING_BINSTAR_TOKEN: ${{ secrets.STAGING_BINSTAR_TOKEN }}
-      if: matrix.os == 'windows'
+
+    - name: Determine build outcome
+      # this is to merge the status of the linux/osx/win builds into
+      # something we can easily reuse during artefact generation
+      id: determine-status
+      if: ${{ always() }}
+      shell: bash
+      env:
+        OS: ${{ matrix.os }}
+      run: |
+        if [[ "$OS" == "ubuntu" ]]; then
+          STATUS="${{ steps.build-linux.outcome }}"
+        elif [[ "$OS" == "macos" ]]; then
+          STATUS="${{ steps.build-macos.outcome }}"
+        elif [[ "$OS" == "windows" ]]; then
+          STATUS="${{ steps.build-windows.outcome }}"
+        fi
+        if [ -z "$STATUS" ]; then
+          # jobs that never ran will have empty status
+          STATUS="cancelled"
+        fi
+        echo "status=$STATUS" >> $GITHUB_OUTPUT
+
     - name: Prepare conda build artifacts
+      continue-on-error: true
       id: prepare-artifacts
       shell: bash
-      if: ${{ always() }}
+      # we do not want to trigger artefact creation if the build was cancelled
+      if: ${{ steps.determine-status.outputs.status != 'cancelled' }}
       env:
+        CI: github_actions
         CONFIG: ${{ matrix.CONFIG }}
         SHORT_CONFIG: ${{ matrix.SHORT_CONFIG }}
+        JOB_STATUS: ${{ steps.determine-status.outputs.status }}
         OS: ${{ matrix.os }}
-        MINIFORGE_HOME_WIN: D:\Miniforge
       run: |
-        export CI=github_actions
         export CI_RUN_ID=$GITHUB_RUN_ID
         export FEEDSTOCK_NAME="$(basename $GITHUB_REPOSITORY)"
         export ARTIFACT_STAGING_DIR="$GITHUB_WORKSPACE"
         if [ $OS == "macos" ]; then
           export CONDA_BLD_DIR="${MINIFORGE_HOME:-${HOME}/miniforge3}/conda-bld"
         elif [ $OS == "windows" ]; then
-          export CONDA_BLD_DIR="${MINIFORGE_HOME_WIN//\\//}/conda-bld"
+          # this should be populated consistently by smithy, see above
+          export CONDA_BLD_DIR="C:\\bld"
         else
           export CONDA_BLD_DIR="build_artifacts"
         fi
         # Archive everything in CONDA_BLD_DIR except environments
         # Archive the CONDA_BLD_DIR environments only when the job fails
         # Use different prefix for successful and failed build artifacts
         # so random failures don't prevent rebuilds from creating artifacts.
-        JOB_STATUS="${{ job.status }}"
         if [ $JOB_STATUS == "failure" ]; then
           export BLD_ARTIFACT_PREFIX="conda_artifacts"
           export ENV_ARTIFACT_PREFIX="conda_envs"
         else
           export BLD_ARTIFACT_PREFIX="conda_pkgs"
         fi
-        ./.scripts/create_conda_build_artifacts.sh
-      continue-on-error: true
+        if [ $OS == "windows" ]; then
+          pwsh -Command ". '.scripts/create_conda_build_artifacts.bat'"
+        else
+          ./.scripts/create_conda_build_artifacts.sh
+        fi
 
     - name: Store conda build artifacts
       uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
-      if: ${{ always() && steps.prepare-artifacts.outcome == 'success' }}
+      if: ${{ steps.prepare-artifacts.outcome == 'success' }}
       with:
         name: ${{ steps.prepare-artifacts.outputs.BLD_ARTIFACT_NAME }}
         path: ${{ steps.prepare-artifacts.outputs.BLD_ARTIFACT_PATH }}
@@ -194,7 +229,8 @@ jobs:
 
     - name: Store conda build environment artifacts
       uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
-      if: ${{ failure() && steps.prepare-artifacts.outcome == 'success' }}
+      # only relevant if build failed, see above
+      if: ${{ steps.determine-status.outputs.status == 'failure' && steps.prepare-artifacts.outcome == 'success' }}
       with:
         name: ${{ steps.prepare-artifacts.outputs.ENV_ARTIFACT_NAME }}
         path: ${{ steps.prepare-artifacts.outputs.ENV_ARTIFACT_PATH }}

.scripts/create_conda_build_artifacts.bat

@@ -2,9 +2,6 @@ azure:
   free_disk_space: true
   settings_linux:
     timeoutInMinutes: 1
-  settings_win:
-    variables:
-      CONDA_BLD_PATH: C:\\bld\\
 bot:
   abi_migration_branches:
   - v2.6.x

conda-forge.yml

@@ -72,6 +72,8 @@ source:
 
 build:
   number: {{ build }}
+  # debug: only rebuild variants that failed
+  skip: true  # [not (cuda_compiler_version != "None" and (win64 or aarch64))]
   # This logic allows two rc variants to be defined in the conda_build_config, but only one to actually be built.
   # We want to be able to define two variants in the cbc so we can assign different labels to each in the upload channel
   # (by zipping is_rc with channel_targets). This prevents rc builds being used unless specifically requested.
@@ -446,6 +448,12 @@ outputs:
         {% set skips = skips ~ " or (TestLinalgCPU and test_pca_lowrank_cpu)" %}        # [aarch64]
         {% set skips = skips ~ " or (TestLinalgCPU and test_svd_lowrank_cpu)" %}        # [aarch64]
         {% set skips = skips ~ " or (TestMkldnnCPU and test_lstm_cpu)" %}               # [aarch64]
+        # very long-running tests in emulation
+        {% set skips = skips ~ " or test_eigh_lwork_lapack" %}                          # [aarch64]
+        {% set skips = skips ~ " or test_gradgrad_nn_LSTM" %}                           # [aarch64]
+        {% set skips = skips ~ " or test_grad_nn_Transformer" %}                        # [aarch64]
+        {% set skips = skips ~ " or test_inverse_errors_large" %}                       # [aarch64]
+        {% set skips = skips ~ " or (TestXNNPACKConv1dTransformPass and test_conv1d_basic)" %}  # [aarch64]
         # errors (possibly QEMU-related) with openblas 0.3.30
         {% set skips = skips ~ " or test_addbmm or test_baddbmm or test_bmm" %}         # [aarch64]
         # doesn't crash, but gets different result on aarch + CUDA