From 5df627a143980e6f0678d4ed59034f549b6a49a1 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Thu, 24 Oct 2024 13:23:03 +0200 Subject: [PATCH 1/3] gh-actions: build wheels for MS Windows as well --- .github/workflows/wheels.yml | 4 ++-- pyproject.toml | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 5cf901b7..67c1d8d4 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -83,13 +83,13 @@ jobs: name: artifact-source path: dist/*.tar.gz - build_wheels_macos: + build_wheels_nonlinux: name: Build wheels on ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: matrix: # macos-13 is an intel runner, macos-14 is apple silicon - os: [macos-13, macos-14] + os: [macos-13, macos-14, windows-latest] steps: - uses: actions/checkout@v4 if: ${{ github.event_name != 'repository_dispatch' }} diff --git a/pyproject.toml b/pyproject.toml index 155a8210..29d7885d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,10 @@ skip = "pp*" build-verbosity = "1" environment = { SCHEMA_SALAD_USE_MYPYC="1", MYPYPATH="$(pwd)/mypy-stubs" } +[tool.cibuildwheel.windows] +test-command = "python -m pytest -n 2 --junitxml={project}/test-results/junit_windows.xml --pyargs schema_salad" +environment = { SCHEMA_SALAD_USE_MYPYC="1", MYPYPATH="%{CD}\\mypy-stubs" } + [tool.black] line-length = 100 target-version = [ "py39" ] From c90fce453d632de13d7734580b017c2925328687 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Thu, 24 Oct 2024 14:09:36 +0200 Subject: [PATCH 2/3] DO NOT MERGE, skip some CI --- .circleci/config.yml | 104 ----------------- .github/workflows/ci-tests.yml | 201 --------------------------------- .github/workflows/wheels.yml | 91 +-------------- 3 files changed, 1 insertion(+), 395 deletions(-) delete mode 100644 .circleci/config.yml delete mode 100644 .github/workflows/ci-tests.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 286c7551..00000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,104 +0,0 @@ -version: 2.1 - -parameters: - REF: - type: string - default: '' - description: Optional tag to build - -jobs: - arm-wheels: - parameters: - build: - type: string - image: - type: string - - machine: - image: ubuntu-2204:current - resource_class: arm.medium # two vCPUs - - environment: - CIBW_ARCHS: "aarch64" - CIBW_MANYLINUX_AARCH64_IMAGE: "<< parameters.image >>" - CIBW_MUSLLINUX_AARCH64_IMAGE: "<< parameters.image >>" - CIBW_BUILD: "<< parameters.build >>" - - steps: - - checkout - - when: - condition: << pipeline.parameters.REF >> - steps: - - run: - name: Checkout branch/tag << pipeline.parameters.REF >> - command: | - echo "Switching to branch/tag << pipeline.parameters.REF >> if it exists" - git checkout << pipeline.parameters.REF >> || true - git pull origin << pipeline.parameters.REF >> || true - - run: - name: install cibuildwheel and other build reqs - command: | - python3 -m pip install --upgrade pip setuptools setuptools_scm[toml] - python3 -m pip install -rcibw-requirements.txt - - - run: - name: pip freeze - command: | - python3 -m pip freeze - - - run: - name: list wheels - command: | - python3 -m cibuildwheel . --print-build-identifiers - - - run: - name: cibuildwheel - command: | - python3 -m cibuildwheel . - - - store_test_results: - path: test-results/ - - - store_artifacts: - path: wheelhouse/ - - - when: - condition: - or: - - matches: - pattern: ".+" - value: "<< pipeline.git.tag >>" - - << pipeline.parameters.REF >> - steps: - - run: - environment: - TWINE_NONINTERACTIVE: "1" - command: | - python3 -m pip install twine - python3 -m twine upload --verbose --skip-existing wheelhouse/* - -workflows: - wheels: # This is the name of the workflow, feel free to change it to better match your workflow. - # Inside the workflow, you define the jobs you want to run. - jobs: - - arm-wheels: - name: arm-wheels-manylinux_2_28 - filters: - tags: - only: /.*/ - build: "*manylinux*" - image: quay.io/pypa/manylinux_2_28_aarch64 - - arm-wheels: - name: arm-wheels-musllinux_1_1 - filters: - tags: - only: /.*/ - build: "*musllinux*" - image: quay.io/pypa/musllinux_1_1_aarch64 - - arm-wheels: - name: arm-wheels-musllinux_1_2 - filters: - tags: - only: /.*/ - build: "*musllinux*" - image: quay.io/pypa/musllinux_1_2_aarch64 diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml deleted file mode 100644 index 7d400cd5..00000000 --- a/.github/workflows/ci-tests.yml +++ /dev/null @@ -1,201 +0,0 @@ -name: Continuous integration tests - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -concurrency: - group: build-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - - tox: - name: CI tests via Tox - - runs-on: ubuntu-24.04 - - strategy: - matrix: - py-ver-major: [3] - py-ver-minor: [9, 10, 11, 12, 13] - fail-fast: false - - env: - py-semver: ${{ format('{0}.{1}', matrix.py-ver-major, matrix.py-ver-minor) }} - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ env.py-semver }} - allow-prereleases: true - cache: pip - cache-dependency-path: | - requirements.txt - tox.ini - - - name: Upgrade setuptools and install tox - run: | - pip install -U pip setuptools wheel - pip install 'tox<4' tox-gh-actions - - - name: MyPy cache - uses: actions/cache@v4 - with: - path: .mypy_cache/${{ env.py-semver }} - key: mypy-${{ env.py-semver }} - - - name: Test with tox - run: tox - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v4 - with: - fail_ci_if_error: true - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - - tox-style: - name: CI linters via Tox - - runs-on: ubuntu-24.04 - - strategy: - matrix: - step: [lintreadme, pydocstyle] - - env: - py-semver: "3.12" - TOXENV: ${{ matrix.step }} - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ env.py-semver }} - cache: pip - - - name: Upgrade setuptools and install tox - run: | - pip install -U pip setuptools wheel - pip install 'tox<4' tox-gh-actions - - - if: ${{ matrix.step == 'pydocstyle' && github.event_name == 'pull_request'}} - name: Create local branch for diff-quality for PRs - run: git branch ${{github.base_ref}} origin/${{github.base_ref}} - - - name: Test with tox - run: tox - - typescript: - name: Confirm that codegen typescript passes tests with CWL - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - - name: Set up Node - uses: actions/setup-node@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: 3.12 - cache: pip - cache-dependency-path: | - requirements.txt - tox.ini - - name: Upgrade setuptools and install schema-salad - run: | - pip install -U pip setuptools wheel - pip install . - - name: Download valid CWL documents for testing - run: | - wget "https://github.com/common-workflow-lab/cwl-ts-auto/archive/refs/heads/main.zip" - unzip main.zip - - name: Generate cwl-ts-auto - run: | - schema-salad-tool --codegen typescript --codegen-examples cwl-ts-auto-main/src/test/data/examples \ - --codegen-target cwl-ts-auto \ - https://github.com/common-workflow-language/cwl-v1.2/raw/codegen/CommonWorkflowLanguage.yml - - name: run cwl-ts-auto tests - run: | - cd cwl-ts-auto - npm install - npm test - - cpp: - name: Confirm that codegen C++ passes tests with CWL - runs-on: ubuntu-24.04 - steps: - - name: Install C++ dependencies - run: sudo apt-get install libyaml-cpp-dev - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: 3.12 - cache: pip - cache-dependency-path: | - requirements.txt - tox.ini - - name: Upgrade setuptools and install schema-salad - run: | - pip install -U pip setuptools wheel - pip install . - - name: Download cwl-cpp-auto for its tests - run: | - wget "https://github.com/common-workflow-lab/cwl-cpp-auto/archive/refs/heads/main.zip" - unzip main.zip - - name: Generate cwl-cpp-auto and run tests - run: | - cd cwl-cpp-auto-main - make cwl_v1_2.h tests CXXFLAGS=-std=c++17 - - release_test: - name: Schema Salad release test - - runs-on: ubuntu-24.04 - - steps: - - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - - name: Cache for pip - uses: actions/cache@v4 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-release-${{ hashFiles('requirements.txt', 'test-requirements.txt') }} - - - name: Install packages - run: | - pip install -U pip setuptools wheel - pip install virtualenv - - - name: Release test - env: - RELEASE_SKIP: head - run: ./release-test.sh - - build_test_container: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - - name: record schema-salad version - run: | - python3 -m venv env - source env/bin/activate && pip install -U setuptools wheel && pip install setuptools_scm[toml] && python setup.py --version - - name: build & test schema_salad container - run: ./build-schema_salad-docker.sh diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 67c1d8d4..7e60579a 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -15,73 +15,6 @@ concurrency: cancel-in-progress: true jobs: - build_wheels: - name: ${{ matrix.image }} wheels - runs-on: ubuntu-24.04 - strategy: - matrix: - include: - - image: manylinux_2_28_x86_64 - build: "*manylinux*" - - image: musllinux_1_1_x86_64 - build: "*musllinux*" - - image: musllinux_1_2_x86_64 - build: "*musllinux*" - - steps: - - uses: actions/checkout@v4 - if: ${{ github.event_name != 'repository_dispatch' }} - with: - fetch-depth: 0 # slow, but gets all the tags - - uses: actions/checkout@v4 - if: ${{ github.event_name == 'repository_dispatch' }} - with: - fetch-depth: 0 # slow, but gets all the tags - ref: ${{ github.event.client_payload.ref }} - - # - name: Set up QEMU - # if: runner.os == 'Linux' - # uses: docker/setup-qemu-action@v2 - # with: - # platforms: all - - - name: Build wheels - uses: pypa/cibuildwheel@v2.21.3 - env: - CIBW_BUILD: ${{ matrix.build }} - CIBW_MANYLINUX_X86_64_IMAGE: quay.io/pypa/${{ matrix.image }} - CIBW_MUSLLINUX_X86_64_IMAGE: quay.io/pypa/${{ matrix.image }} - # configure cibuildwheel to build native 64-bit archs ('auto64'), and some - # emulated ones - # Linux arm64 wheels are built on circleci - CIBW_ARCHS_LINUX: auto64 # ppc64le s390x - - - uses: actions/upload-artifact@v4 - with: - name: artifact-${{ matrix.image }} - path: ./wheelhouse/*.whl - - build_sdist: - name: Build source distribution - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - if: ${{ github.event_name != 'repository_dispatch' }} - with: - fetch-depth: 0 # slow, but gets all the tags - - uses: actions/checkout@v4 - if: ${{ github.event_name == 'repository_dispatch' }} - with: - fetch-depth: 0 # slow, but gets all the tags - ref: ${{ github.event.client_payload.ref }} - - - name: Build sdist - run: pipx run build --sdist - - - uses: actions/upload-artifact@v4 - with: - name: artifact-source - path: dist/*.tar.gz build_wheels_nonlinux: name: Build wheels on ${{ matrix.os }} @@ -89,7 +22,7 @@ jobs: strategy: matrix: # macos-13 is an intel runner, macos-14 is apple silicon - os: [macos-13, macos-14, windows-latest] + os: [windows-latest] steps: - uses: actions/checkout@v4 if: ${{ github.event_name != 'repository_dispatch' }} @@ -108,25 +41,3 @@ jobs: with: name: artifact-${{ matrix.os }}-${{ strategy.job-index }} path: ./wheelhouse/*.whl - - upload_pypi: - needs: [build_wheels, build_sdist] - runs-on: ubuntu-24.04 - environment: pypi - permissions: - id-token: write - if: (github.event_name == 'release' && github.event.action == 'published') || (github.event_name == 'repository_dispatch' && github.event.client_payload.publish_wheel == true) - steps: - - uses: actions/download-artifact@v4 - with: - # unpacks default artifact into dist/ - pattern: artifact-* - merge-multiple: true - path: dist - - - uses: pypa/gh-action-pypi-publish@release/v1 - with: - # To test: repository-url: https://test.pypi.org/legacy/ - user: __token__ - password: ${{ secrets.PYPI_API_TOKEN }} - skip-existing: true From bfd21b4b4002ab0fdc75814f0c7781347d7ef700 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Thu, 24 Oct 2024 14:16:24 +0200 Subject: [PATCH 3/3] try again --- pyproject.toml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 29d7885d..3cca759f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,11 @@ environment = { SCHEMA_SALAD_USE_MYPYC="1", MYPYPATH="$(pwd)/mypy-stubs" } [tool.cibuildwheel.windows] test-command = "python -m pytest -n 2 --junitxml={project}/test-results/junit_windows.xml --pyargs schema_salad" -environment = { SCHEMA_SALAD_USE_MYPYC="1", MYPYPATH="%{CD}\\mypy-stubs" } +environment = { SCHEMA_SALAD_USE_MYPYC="1", MYPYPATH="mypy-stubs" } +before-build = [ + "echo MYPYPATH=%MYPYPATH%", + "SET" +] [tool.black] line-length = 100