release.yml

name: Release

on:
  workflow_dispatch:
  push:
    branches: ['*']
    tags: ['*']
    paths-ignore:
      - 'docs/**'
  pull_request:
    branches: [main]
    paths-ignore:
      - 'docs/**'

permissions:
  # needs write permission at workflow level to create release and upload artifacts
  contents: write

env:
  MPLBACKEND: agg
  QT_API: pyqt5
  # pytest-qt doesn't seem to be playing well when pyqt5 and pyside6
  # are installed in the same environment
  PYTEST_QT_API: pyqt5
  # workaround for https://github.com/pyxem/pyxem/pull/1175
  TEST_DEPS: '"pytest<9" pytest-qt pytest-xdist pytest-rerunfailures pytest-mpl filelock'
  CONSTRUCTOR_VERSION: '3.14.0'
  CONDA_STANDALONE: '24.11.0'
  ROSETTASCIIO_CACHE_NUMBER: 0 # Increase this value to reset cache

defaults:
  run:
    shell: bash -el {0}

jobs:
  create_release_job:
    name: Create Release (on tag only)
    runs-on: ubuntu-latest
    outputs: 
      ID: ${{ steps.create_release.outputs.id }}
      VERSION: ${{ env.VERSION }}
    steps:
      - uses: actions/checkout@v6

      - name: Get version (on tag)
        if: startsWith(github.ref, 'refs/tags/')
        run: |
          echo "VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV

      - name: Get version (short hash)
        if: ${{ startsWith(github.ref, 'refs/tags/') == false }}
        run: |
          # Use git short hash instead of tag
          echo "VERSION=$(git rev-parse --short HEAD)" >> $GITHUB_ENV

      - name: Display version
        run: |
          echo ${{ env.VERSION }}

      - uses: conda-incubator/setup-miniconda@v3
        with:
            miniforge-version: latest

      - name: Install constructor
        run: |
          # Constructor is needed to parse the list of included packages
          conda install constructor==${{ env.CONSTRUCTOR_VERSION }}

      - name: Conda info
        run: |
          conda info
          conda list

      - name: Find and Replace Version in Release Notes
        uses: jacobtomlinson/gha-find-replace@f1069b438f125e5395d84d1c6fd3b559a7880cb5
        if: startsWith(github.ref, 'refs/tags/')
        with:
          find: "__TAG__"
          replace: ${{ env.VERSION }}
          include: "RELEASE_TEXT.md" # Will match all RELEASE_TEXT.md files in any nested directory

      - name: Find and Replace constructor version in Release Notes
        uses: jacobtomlinson/gha-find-replace@f1069b438f125e5395d84d1c6fd3b559a7880cb5
        if: startsWith(github.ref, 'refs/tags/')
        with:
          find: "__CONSTRUCTOR_VERSION__"
          replace: ${{ env.CONSTRUCTOR_VERSION }}
          include: "RELEASE_TEXT.md" # Will match all RELEASE_TEXT.md files in any nested directory

      - name: Find and Replace conda-standalone version in Release Notes
        uses: jacobtomlinson/gha-find-replace@f1069b438f125e5395d84d1c6fd3b559a7880cb5
        if: startsWith(github.ref, 'refs/tags/')
        with:
          find: "__CONDA_STANDALONE_VERSION__"
          replace: ${{ env.CONDA_STANDALONE }}
          include: "RELEASE_TEXT.md" # Will match all RELEASE_TEXT.md files in any nested directory

      - name: Update packages list
        run: |
          python parse_list_libraries.py
          echo "List of libraries and software"
          echo "=============================="
          cat ./docs/specs.rst

      - name: Save `docs/specs.rst` content as environment variable
        run: |
          echo "INCLUDED_PACKAGES<<EOF" >> $GITHUB_ENV
          cat ./docs/specs.rst >> $GITHUB_ENV
          echo "EOF" >> $GITHUB_ENV

      - name: Find and Replace included packages in Release Notes
        uses: jacobtomlinson/gha-find-replace@f1069b438f125e5395d84d1c6fd3b559a7880cb5
        if: startsWith(github.ref, 'refs/tags/')
        with:
          find: "__INCLUDED_PACKAGES__"
          replace: ${{ env.INCLUDED_PACKAGES }}
          include: "RELEASE_TEXT.md" # Will match all RELEASE_TEXT.md files in any nested directory

      - name: Create Release
        if: startsWith(github.ref, 'refs/tags/')
        id: create_release
        uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions
        with:
          body_path: "RELEASE_TEXT.md"
          draft: true


  build:
    name: ${{ matrix.TARGET_PLATFORM }}-${{ matrix.BLAS_IMPL }}
    needs: create_release_job
    runs-on: ${{ matrix.os }}-${{ matrix.os_version }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - os: ubuntu
            os_version: latest
            INSTALLER_EXTENSION: sh
            BLAS_IMPL: mkl
            ARCH: x86_64
            TARGET_PLATFORM: linux-64
          - os: ubuntu
            os_version: latest
            INSTALLER_EXTENSION: sh
            BLAS_IMPL: openblas
            ARCH: x86_64
            TARGET_PLATFORM: linux-64
          - os: windows
            os_version: latest
            INSTALLER_EXTENSION: exe
            BLAS_IMPL: mkl
            ARCH: x86_64
            TARGET_PLATFORM: win-64
          - os: windows
            os_version: latest
            INSTALLER_EXTENSION: exe
            BLAS_IMPL: openblas
            ARCH: x86_64
            TARGET_PLATFORM: win-64
          - os: macos
            os_version: '15-intel'
            INSTALLER_EXTENSION: pkg
            BLAS_IMPL: mkl
            ARCH: x86_64
            TARGET_PLATFORM: osx-64
          - os: macos
            os_version: latest
            INSTALLER_EXTENSION: pkg
            BLAS_IMPL: newaccelerate
            ARCH: arm64
            TARGET_PLATFORM: osx-arm64
    env:
      DISPLAY: ':0'

    steps:
      - uses: actions/checkout@v6
      - uses: conda-incubator/setup-miniconda@v3
        with:
          miniforge-version: latest

      - name: Install constructor
        run: |
          conda install constructor==${{ env.CONSTRUCTOR_VERSION }} conda-standalone==${{ env.CONDA_STANDALONE }}

      - name: Conda info
        run: |
          conda info
          conda list

      - name: Build distribution
        env:
          VERSION: ${{ needs.create_release_job.outputs.VERSION }}
          BLAS_IMPL: ${{ matrix.BLAS_IMPL }}
          CONDA_SOLVER: 'libmamba'
        run: |
          constructor -v conda_distribution

      - name: Clean constructor cache
        if: runner.os != 'windows'
        # PermissionError on windows
        run: |
          constructor --clean

      - name: Set asset name
        env:
          ext: ${{ matrix.INSTALLER_EXTENSION }}
        run: |  
          installer_name=(HyperSpy-*.${{ env.ext }})
          echo "Original installer name: " $installer_name
          # Rename installer name according to BLAS_IMPL
          if [ ${{ matrix.TARGET_PLATFORM }} = osx-arm64 ]; then
             CPU=Silicon
          elif [ ${{ matrix.BLAS_IMPL }} = mkl ]; then
             CPU=Intel
          else
             CPU=AMD
          fi    
          new_installer_name=$(basename $installer_name .${{ env.ext }})-$CPU.${{ env.ext }}   
          mv $installer_name $new_installer_name
          echo "asset_name=$new_installer_name" >> $GITHUB_ENV
          echo "Installer name: "$new_installer_name
          ls

      - name: Get hash
        run: |
          shasum -a 256 ${{ env.asset_name }}

      - name: Install new distribution (Linux)
        if: runner.os == 'linux'
        env:
          install_dir: '${{ github.workspace }}/new_distribution'
        run: |
          echo "install_dir=${{ env.install_dir }}" >> $GITHUB_ENV
          bash ${{ env.asset_name }} -b -p ${{ env.install_dir }}

      - name: Install new distribution (MacOS)
        if: runner.os == 'macos'
        run: |
          echo "install_dir=/Users/runner/hyperspy-bundle" >> $GITHUB_ENV
          installer -pkg ${{ env.asset_name }} -target CurrentUserHomeDirectory

      - name: Install new distribution (Windows)
        if: runner.os == 'windows'
        env:
          install_dir: '${{ github.workspace }}\nd'
        run: |
          echo "install_dir=${{ env.install_dir }}" >> $GITHUB_ENV

      - name: Install new distribution (Windows)
        if: runner.os == 'windows'
        shell: powershell
        run: |
          Start-Process -Wait -FilePath ${{ env.asset_name }} -ArgumentList "/S /AddToPath=0 /RegisterPython=0 /NoScripts=1 /D=${{ env.install_dir }}"

      - name: Upload artifact
        if: startsWith(github.ref, 'refs/tags/') != true
        uses: actions/upload-artifact@v5
        with:
          path: ${{ env.asset_name }}
          name: ${{ env.asset_name }}

      - name: Show install folder content
        run : |
          ls "${{ env.install_dir }}"

      - name: Info new distribution
        run: |
          conda activate "${{ env.install_dir }}"
          conda info
          conda config --show channels
          conda config --show channel_priority
          conda list

      - name: Check latest available hyperspy version
        run: |
          conda activate "${{ env.install_dir }}"
          python check_hyperspy_latest.py

      - name: Install xvfb
        if: runner.os == 'linux' && always()
        run: |
          sudo apt-get install -y xvfb libxkbcommon-x11-0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-randr0 libxcb-render-util0 libxcb-xinerama0 libxcb-xinput0 libxcb-xfixes0

      - name: Start xvfb
        if: runner.os == 'linux' && always()
        run: |
          sudo /usr/bin/Xvfb ${{ env.DISPLAY }} -screen 0 1280x1024x24 & 

      - name: Install test dependencies
        if: always()
        run: |
          conda activate "${{ env.install_dir }}"
          mamba install ${{ env.TEST_DEPS }}

      - name: Get rosettasciio tests data location
        run: |
          conda activate "${{ env.install_dir }}"
          python -c "from rsciio.tests import registry; print(f'ROSETTASCIIO_TEST_DATA={str(registry.TESTS_PATH / \"data\") }')" >> $GITHUB_ENV

      - name: Cache rosettasciio data
        uses: actions/cache@v4
        with:
          path: ${{ env.ROSETTASCIIO_TEST_DATA }}
          key:
            RosettaSciIO-test_data-${{ env.ROSETTASCIIO_CACHE_NUMBER }}
          restore-keys: |
            RosettaSciIO-test_data-${{ env.ROSETTASCIIO_CACHE_NUMBER }}

      - if: always()
        name: Test new distribution (HyperSpyUI)
        run: |
          conda activate "${{ env.install_dir }}"
          # Error on closing figure, not critical
          pytest --pyargs hyperspyui -k "not plotting"

      - name: Test RosettaSciIO
        if: always()
        run: |
          conda activate "${{ env.install_dir }}"
          pytest --pyargs rsciio --reruns 3 -n 1

      - name: Test hyperspy
        if: always()
        run: |
          conda activate "${{ env.install_dir }}"
          # test_rank_lstsq_residual fails on windows with MKL
          # map_values_std_isset fixed in https://github.com/hyperspy/hyperspy/pull/3554
          pytest --pyargs hyperspy --reruns 3 -n 2 -k "not test_arpack and not map_values_std_isset"

      - name: Test hyperspy_gui_ipywidgets
        if: always()
        run: |
          conda activate "${{ env.install_dir }}"
          pytest --pyargs hyperspy_gui_ipywidgets

      - name: Test hyperspy_gui_traitsui
        if: runner.os != 'linux' && always()
        run: |
          conda activate "${{ env.install_dir }}"
          pytest --pyargs hyperspy_gui_traitsui

      - name: Run test exspy
        if: always()
        run: |
          conda activate "${{ env.install_dir }}"
          pytest --pyargs exspy
  
      - name: Run test holospy
        if: always()
        run: |
          conda activate "${{ env.install_dir }}"
          pytest --pyargs holospy

      - name: Run test lumispy
        if: always()
        run: |
          conda activate "${{ env.install_dir }}"
          pytest --pyargs lumispy

      - name: Run test pyxem
        if: always()
        run: |
          conda activate "${{ env.install_dir }}"
          # cause of the TestAddEllipseArrayAsMarkers failure is unknown (seems to be on blas mkl only)
          pytest --pyargs pyxem -k "not TestAddEllipseArrayAsMarkers and not TestGetEllipseModelRansacSingleFrame"

      - name: Run test atomap
        if: always()
        run: |
          conda activate "${{ env.install_dir }}"
          # Skip interactive test
          pytest --pyargs atomap -k "not TestSelectAtomsWithGui"

      - name: Run test kikuchipy
        if: always()
        run: |
          conda activate "${{ env.install_dir }}"
          # https://github.com/pyxem/kikuchipy/issues/707
          # https://github.com/pyxem/kikuchipy/pull/749
          pytest --pyargs kikuchipy -k "not test_not_allow_download_raises and not spherical"

      - name: Upload Release Asset
        if: startsWith(github.ref, 'refs/tags/')
        uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          files: ${{ env.asset_name }}
          draft: true


  build_portable:
    name: windows (Portable)
    needs: create_release_job
    runs-on: windows-latest
    env:
      WP_URL: https://github.com/winpython/winpython/releases/download/11.2.20241228final/Winpython64-3.12.8.0slim.exe
      WP_SHA256: c4c59403370e62169bbeba06f140f0da76b2ee5a5a51ebcfdb3d3ad663b2688a
      WP_EXE: winpython.exe
      WP_DIR_NAME: WPy64-31280

    defaults:
      run:
        shell: cmd

    steps:
      - uses: actions/checkout@v6

      - name: Cache WinPython data
        uses: actions/cache@v4
        env:
          # Increase this value to reset cache
          CACHE_NUMBER: 0
        with:
          path: ${{ env.WP_EXE }}
          key:
            WinPython-${{ env.WP_DIR_NAME }}-${{ env.CACHE_NUMBER }}
          restore-keys: |
            WinPython-${{ env.WP_DIR_NAME }}-${{ env.CACHE_NUMBER }}

      - name: Download Winpython
        # only run if not in cache
        if: hashFiles(env.WP_EXE) == ''
        shell: powershell
        run: |
          Invoke-WebRequest -OutFile ${{ env.WP_EXE }} ${{ env.WP_URL }}
          ls
          $file_hash = (Get-FileHash ${{ env.WP_EXE }} ).Hash
          $file_hash 
          if ($file_hash -ne "${{ env.WP_SHA256 }}") { exit(1) }

      - name: Install Winpython
        shell: powershell
        run: |
          .\${{ env.WP_EXE }} -y | Out-Null
          ls
          ls ${{ env.WP_DIR_NAME }}

      - name: Install libraries
        run: |
          call "${{ env.WP_DIR_NAME }}\scripts\env.bat"
          where python
          where pip
          pip install -r requirement_portable_distribution.txt

      - name: Pip list
        run: |
          call "${{ env.WP_DIR_NAME }}\scripts\env.bat"
          pip list

      - name: Set installer name
        shell: bash -l {0}
        env:
          VERSION: ${{ needs.create_release_job.outputs.VERSION }}
        run: |
          installer_name=HyperSpy-bundle-${{ env.VERSION }}-Windows-x86_64-Portable.exe
          echo "asset_name=$installer_name" >> $GITHUB_ENV
          echo $installer_name

      - name: Run pyclean in distribution folder
        run: |
          where pip
          pip install pyclean
          where pyclean
          pyclean ${{ env.WP_DIR_NAME }}

      - name: Create installer
        run: |
          7z -mx5 -sfx a ${{ env.asset_name }} ${{ env.WP_DIR_NAME }}
          ls

      - name: Upload artifact
        if: startsWith(github.ref, 'refs/tags/') != true
        uses: actions/upload-artifact@v5
        with:
          path: ${{ env.asset_name }}
          name: ${{ env.asset_name }}

      - name: Install new distribution
        shell: powershell
        run: |
          Remove-Item -recurse ${{ env.WP_DIR_NAME }}
          ls
          .\${{ env.asset_name }} -y | Out-Null
          echo "After installation"
          ls
          echo ${{ env.asset_name }}

      - name: Check latest available hyperspy version
        run: |
          call "${{ env.WP_DIR_NAME }}\scripts\env.bat"
          python check_hyperspy_latest.py

      - name: Get rosettasciio tests data location
        run: |
          call "${{ env.WP_DIR_NAME }}\scripts\env.bat"
          python -c "from rsciio.tests import registry; data_path=str(registry.TESTS_PATH / 'data'); print(f'ROSETTASCIIO_TEST_DATA={data_path}')">> %GITHUB_ENV%

      - name: Cache rosettasciio data
        uses: actions/cache@v4
        with:
          path: ${{ env.ROSETTASCIIO_TEST_DATA }}
          key:
            RosettaSciIO-test_data-${{ env.ROSETTASCIIO_CACHE_NUMBER }}
          restore-keys: |
            RosettaSciIO-test_data-${{ env.ROSETTASCIIO_CACHE_NUMBER }}

      - name: Install testing libraries
        if: always()
        run: |
          call "${{ env.WP_DIR_NAME }}\scripts\env.bat"
          pip install ${{ env.TEST_DEPS }}

      - name: Run test suite (HyperSpyUI)
        if: always()
        run: |
          call "${{ env.WP_DIR_NAME }}\scripts\env.bat"
          # Error on closing figure, not critical
          pytest --pyargs hyperspyui -k "not plotting"

      - name: Run test hyperspy
        if: always()
        run: |
          call "${{ env.WP_DIR_NAME }}\scripts\env.bat"
          pytest --pyargs hyperspy --reruns 3 -n 2

      - name: Run test RosettaSciIO
        if: always()
        run: |
          call "${{ env.WP_DIR_NAME }}\scripts\env.bat"
          pytest --pyargs rsciio --reruns 3 -n 2 -k "not test_mrcz.py and not test_usid.py"

      - name: Run test hyperspy_gui_ipywidgets
        if: always()
        run: |
          call "${{ env.WP_DIR_NAME }}\scripts\env.bat"
          pytest --pyargs hyperspy_gui_ipywidgets

      - name: Run test hyperspy_gui_traitsui
        if: always()
        run: |
          call "${{ env.WP_DIR_NAME }}\scripts\env.bat"
          pytest --pyargs hyperspy_gui_traitsui

      - name: Run test exspy
        if: always()
        run: |
          call "${{ env.WP_DIR_NAME }}\scripts\env.bat"
          pytest --pyargs exspy

      - name: Run test holospy
        if: always()
        run: |
          call "${{ env.WP_DIR_NAME }}\scripts\env.bat"
          pytest --pyargs holospy

      - name: Run test lumispy
        if: always()
        run: |
          call "${{ env.WP_DIR_NAME }}\scripts\env.bat"
          pytest --pyargs lumispy

      - name: Run test pyxem
        if: always()
        run: |
          call "${{ env.WP_DIR_NAME }}\scripts\env.bat"
          # cause of the TestAddEllipseArrayAsMarkers failure is unknown (seems to be on blas mkl only)
          pytest --pyargs pyxem -k "not TestAddEllipseArrayAsMarkers and not TestGetEllipseModelRansacSingleFrame"
 
      - name: Run test atomap
        if: always()
        run: |
          call "${{ env.WP_DIR_NAME }}\scripts\env.bat"
          # Skip interactive test
          pytest --pyargs atomap -k "not TestSelectAtomsWithGui"

      - name: Run test kikuchipy
        if: always()
        run: |
          call "${{ env.WP_DIR_NAME }}\scripts\env.bat"
          # https://github.com/pyxem/kikuchipy/issues/707
          # https://github.com/pyxem/kikuchipy/pull/749
          pytest --pyargs kikuchipy -k "not test_not_allow_download_raises and not spherical"

      - name: Upload Release Asset
        if: startsWith(github.ref, 'refs/tags/')
        uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          files: ${{ env.asset_name }}
          draft: true

  publish_release_job:
    # Set build, build_portable are needed, so that it runs when they are finished 
    needs: [create_release_job, build, build_portable]
    if: startsWith(github.ref, 'refs/tags/')
    runs-on: ubuntu-latest
    steps:
      - name: Finalise release
        # Publish draft release
        uses: eregon/publish-release@01df127f5e9a3c26935118e22e738d95b59d10ce
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          release_id: ${{ needs.create_release_job.outputs.ID }}