diff --git a/.github/dependabot.yml b/.github/dependabot.yml index bd88b693..1a706cd6 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -13,23 +13,43 @@ updates: versioning-strategy: auto # Allow up to 10 open pull requests for updates to dependency versions open-pull-requests-limit: 10 - # We group production and development ("optional" in the context of - # pyproject.toml) dependency updates when they are patch and minor updates, - # so we end up with less PRs being generated. - # Major updates are still managed, but they'll create one PR per - # dependency, as major updates are expected to be breaking, it is better to - # manage them individually. + # We group patch updates as they should always work. + # We also group minor updates, as it works too for most libraries, + # typically except libraries that don't have a stable release yet (v0.x.x + # branch), so we make some exceptions for them. + # Major updates and dependencies excluded by the above groups are still + # managed, but they'll create one PR per dependency, as breakage is + # expected, so it might need manual intervention. + # Finally, we group some dependencies that are related to each other, and + # usually need to be updated together. groups: - required: - dependency-type: "production" + patch: update-types: - - "minor" - "patch" - optional: - dependency-type: "development" + exclude-patterns: + # pydoclint has shipped breaking changes in patch updates often + - "pydoclint" + minor: update-types: - "minor" - - "patch" + exclude-patterns: + - "async-solipsism" + - "frequenz-repo-config*" + - "markdown-callouts" + - "mkdocs-gen-files" + - "mkdocs-literate-nav" + - "mkdocstrings*" + - "pydoclint" + - "pytest-asyncio" + # We group repo-config updates as it uses optional dependencies that are + # considered different dependencies otherwise, and will create one PR for + # each if we don't group them. + repo-config: + patterns: + - "frequenz-repo-config*" + mkdocstrings: + patterns: + - "mkdocstrings*" - package-ecosystem: "github-actions" directory: "/" diff --git a/.github/workflows/ci-pr.yaml b/.github/workflows/ci-pr.yaml new file mode 100644 index 00000000..69083d4b --- /dev/null +++ b/.github/workflows/ci-pr.yaml @@ -0,0 +1,81 @@ +name: Test PR + +on: + pull_request: + +env: + # Please make sure this version is included in the `matrix`, as the + # `matrix` section can't use `env`, so it must be entered manually + DEFAULT_PYTHON_VERSION: '3.11' + # It would be nice to be able to also define a DEFAULT_UBUNTU_VERSION + # but sadly `env` can't be used either in `runs-on`. + +jobs: + protolint: + name: Check proto files with protolint + runs-on: ubuntu-24.04 + + steps: + - name: Setup Git + uses: frequenz-floss/gh-action-setup-git@v1.0.0 + + - name: Fetch sources + uses: actions/checkout@v3 + with: + submodules: true + + - name: Run protolint + # Only use hashes here, as we are passing the github token, we want to + # make sure updates are done consciously to avoid security issues if the + # action repo gets hacked + uses: yoheimuta/action-protolint@e62319541dc5107df5e3a5010acb8987004d3d25 # v1.3.0 + with: + fail_on_error: true + filter_mode: nofilter + github_token: ${{ secrets.github_token }} + protolint_flags: proto/ + protolint_version: "0.52.0" + reporter: github-check + + nox: + name: Test with nox + runs-on: ubuntu-24.04 + + steps: + - name: Run nox + uses: frequenz-floss/gh-action-nox@v1.0.0 + with: + python-version: "3.11" + nox-session: ci_checks_max + + test-docs: + name: Test documentation website generation + runs-on: ubuntu-24.04 + steps: + - name: Setup Git + uses: frequenz-floss/gh-action-setup-git@v1.0.0 + + - name: Fetch sources + uses: actions/checkout@v4 + with: + submodules: true + + - name: Setup Python + uses: frequenz-floss/gh-action-setup-python-with-deps@v1.0.0 + with: + python-version: ${{ env.DEFAULT_PYTHON_VERSION }} + dependencies: .[dev-mkdocs] + + - name: Generate the documentation + env: + MIKE_VERSION: gh-${{ github.job }} + run: | + mike deploy $MIKE_VERSION + mike set-default $MIKE_VERSION + + - name: Upload site + uses: actions/upload-artifact@v4 + with: + name: docs-site + path: site/ + if-no-files-found: error diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a76f8b24..503f4f2c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -2,7 +2,6 @@ name: CI on: merge_group: - pull_request: push: # We need to explicitly include tags because otherwise when adding # `branches-ignore` it will only trigger on branches. @@ -13,6 +12,7 @@ on: # We only want to test the merge commit (`merge_group` event), the hashes # in the push were already tested by the PR checks - 'gh-readonly-queue/**' + - 'dependabot/**' workflow_dispatch: env: @@ -25,11 +25,14 @@ env: jobs: protolint: name: Check proto files with protolint - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: + - name: Setup Git + uses: frequenz-floss/gh-action-setup-git@v1.0.0 + - name: Fetch sources - uses: actions/checkout@v4 + uses: actions/checkout@v3 with: submodules: true @@ -43,7 +46,7 @@ jobs: filter_mode: nofilter github_token: ${{ secrets.github_token }} protolint_flags: proto/ - protolint_version: "0.50.5" + protolint_version: "0.52.0" reporter: github-check nox: @@ -51,77 +54,66 @@ jobs: strategy: fail-fast: false matrix: + arch: + - amd64 + - arm os: - - ubuntu-20.04 + - ubuntu-24.04 python: - "3.11" + - "3.12" nox-session: # To speed things up a bit we use the special ci_checks_max session # that uses the same venv to run multiple linting sessions - "ci_checks_max" - "pytest_min" - runs-on: ${{ matrix.os }} + runs-on: ${{ matrix.os }}${{ matrix.arch != 'amd64' && format('-{0}', matrix.arch) || '' }} steps: - - name: Print environment (debug) - run: env - - - name: Fetch sources - uses: actions/checkout@v4 - with: - submodules: true - - - name: Set up Python - uses: actions/setup-python@v5 + - name: Run nox + uses: frequenz-floss/gh-action-nox@v1.0.0 with: python-version: ${{ matrix.python }} - cache: 'pip' - - - name: Install required Python packages - run: | - python -m pip install --upgrade pip - python -m pip install -e .[dev-noxfile] - pip freeze - - - name: Create nox venv - env: - NOX_SESSION: ${{ matrix.nox-session }} - run: nox --install-only -e "$NOX_SESSION" - - - name: Print pip freeze for nox venv (debug) - env: - NOX_SESSION: ${{ matrix.nox-session }} - run: | - . ".nox/$NOX_SESSION/bin/activate" - pip freeze - deactivate - - - name: Run nox - env: - NOX_SESSION: ${{ matrix.nox-session }} - run: nox -R -e "$NOX_SESSION" - timeout-minutes: 10 + nox-session: ${{ matrix.nox-session }} + + # This job runs if all the `nox` matrix jobs ran and succeeded. + # It is only used to have a single job that we can require in branch + # protection rules, so we don't have to update the protection rules each time + # we add or remove a job from the matrix. + nox-all: + # The job name should match the name of the `nox` job. + name: Test with nox + needs: ["nox"] + # We skip this job only if nox was also skipped + if: always() && needs.nox.result != 'skipped' + runs-on: ubuntu-24.04 + env: + DEPS_RESULT: ${{ needs.nox.result }} + steps: + - name: Check matrix job result + run: test "$DEPS_RESULT" = "success" build: name: Build distribution packages - runs-on: ubuntu-20.04 + # Since this is a pure Python package, we only need to build it once. If it + # had any architecture specific code, we would need to build it for each + # architecture. + runs-on: ubuntu-24.04 + steps: + - name: Setup Git + uses: frequenz-floss/gh-action-setup-git@v1.0.0 + - name: Fetch sources uses: actions/checkout@v4 with: submodules: true - - name: Set up Python - uses: actions/setup-python@v5 + - name: Setup Python + uses: frequenz-floss/gh-action-setup-python-with-deps@v1.0.0 with: python-version: ${{ env.DEFAULT_PYTHON_VERSION }} - cache: 'pip' - - - name: Install required Python packages - run: | - python -m pip install -U pip - python -m pip install -U build - pip freeze + dependencies: build - name: Build the source and binary distribution run: python -m build @@ -133,30 +125,93 @@ jobs: path: dist/ if-no-files-found: error + test-installation: + name: Test package installation + needs: ["build"] + strategy: + fail-fast: false + matrix: + arch: + - amd64 + - arm + os: + - ubuntu-24.04 + python: + - "3.11" + - "3.12" + runs-on: ${{ matrix.os }}${{ matrix.arch != 'amd64' && format('-{0}', matrix.arch) || '' }} + + steps: + - name: Setup Git + uses: frequenz-floss/gh-action-setup-git@v1.0.0 + + - name: Print environment (debug) + run: env + + - name: Download package + uses: actions/download-artifact@v4 + with: + name: dist-packages + path: dist + + # This is necessary for the `pip` caching in the setup-python action to work + - name: Fetch the pyproject.toml file for this action hash + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + REF: ${{ github.sha }} + run: | + set -ux + gh api \ + -X GET \ + -H "Accept: application/vnd.github.raw" \ + "/repos/$REPO/contents/pyproject.toml?ref=$REF" \ + > pyproject.toml + + - name: Setup Python + uses: frequenz-floss/gh-action-setup-python-with-deps@v1.0.0 + with: + python-version: ${{ matrix.python }} + dependencies: dist/*.whl + + - name: Print installed packages (debug) + run: python -m pip freeze + + # This job runs if all the `test-installation` matrix jobs ran and succeeded. + # It is only used to have a single job that we can require in branch + # protection rules, so we don't have to update the protection rules each time + # we add or remove a job from the matrix. + test-installation-all: + # The job name should match the name of the `test-installation` job. + name: Test package installation + needs: ["test-installation"] + # We skip this job only if test-installation was also skipped + if: always() && needs.test-installation.result != 'skipped' + runs-on: ubuntu-24.04 + env: + DEPS_RESULT: ${{ needs.test-installation.result }} + steps: + - name: Check matrix job result + run: test "$DEPS_RESULT" = "success" + test-docs: name: Test documentation website generation if: github.event_name != 'push' - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: + - name: Setup Git + uses: frequenz-floss/gh-action-setup-git@v1.0.0 + - name: Fetch sources uses: actions/checkout@v4 with: submodules: true - - name: Setup Git user and e-mail - uses: frequenz-floss/setup-git-user@v2 - - - name: Set up Python - uses: actions/setup-python@v5 + - name: Setup Python + uses: frequenz-floss/gh-action-setup-python-with-deps@v1.0.0 with: python-version: ${{ env.DEFAULT_PYTHON_VERSION }} - cache: 'pip' - - - name: Install build dependencies - run: | - python -m pip install -U pip - python -m pip install .[dev-mkdocs] - pip freeze + dependencies: .[dev-mkdocs] - name: Generate the documentation env: @@ -174,84 +229,68 @@ jobs: publish-docs: name: Publish documentation website to GitHub pages - needs: ["nox", "build", "protolint"] + needs: ["nox-all", "test-installation-all"] if: github.event_name == 'push' - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 permissions: contents: write steps: - - name: Calculate and check version - id: mike-metadata - env: - REF: ${{ github.ref }} - REF_NAME: ${{ github.ref_name }} - DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} - run: | - aliases= - version= - if test "$REF_NAME" = "$DEFAULT_BRANCH" - then - version=next - # A tag that starts with vX.Y or X.Y - elif echo "$REF" | grep -q '^refs/tags' && echo "$REF_NAME" | grep -Pq '^v?\d+\.\d+\.' - then - if echo "$REF_NAME" | grep -Pq -- "-" # pre-release - then - echo "::notice title=Documentation was not published::" \ - "The tag '$REF_NAME' looks like a pre-release." - exit 0 - fi - version=$(echo "$REF_NAME" | sed -r 's/^(v?[0-9]+\.[0-9]+)\..*$/\1/') # vX.Y - major=$(echo "$REF_NAME" | sed -r 's/^(v?[0-9]+)\..*$/\1/') # vX - default_major=$(echo "$DEFAULT_BRANCH" | sed -r 's/^(v?[0-9]+)\..*$/\1/') # vX - aliases=$major - if test "$major" = "$default_major" - then - aliases="$aliases latest" - fi - else - echo "::warning title=Documentation was not published::" \ - "Don't know how to handle '$REF' to make 'mike' version." - exit 0 - fi - echo "version=$version" >> $GITHUB_OUTPUT - echo "aliases=$aliases" >> $GITHUB_OUTPUT + - name: Setup Git + uses: frequenz-floss/gh-action-setup-git@v1.0.0 - name: Fetch sources - if: steps.mike-metadata.outputs.version uses: actions/checkout@v4 with: submodules: true - - name: Setup Git user and e-mail - if: steps.mike-metadata.outputs.version - uses: frequenz-floss/setup-git-user@v2 - - - name: Set up Python - if: steps.mike-metadata.outputs.version - uses: actions/setup-python@v5 + - name: Setup Python + uses: frequenz-floss/gh-action-setup-python-with-deps@v1.0.0 with: python-version: ${{ env.DEFAULT_PYTHON_VERSION }} - cache: 'pip' + dependencies: .[dev-mkdocs] - - name: Install build dependencies - if: steps.mike-metadata.outputs.version + - name: Calculate and check version + id: mike-version + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPO: ${{ github.repository }} + GIT_REF: ${{ github.ref }} + GIT_SHA: ${{ github.sha }} run: | - python -m pip install -U pip - python -m pip install .[dev-mkdocs] - pip freeze + python -m frequenz.repo.config.cli.version.mike.info - name: Fetch the gh-pages branch - if: steps.mike-metadata.outputs.version + if: steps.mike-version.outputs.version run: git fetch origin gh-pages --depth=1 - - name: Publish site - if: steps.mike-metadata.outputs.version + - name: Build site + if: steps.mike-version.outputs.version env: - VERSION: ${{ steps.mike-metadata.outputs.version }} - ALIASES: ${{ steps.mike-metadata.outputs.aliases }} + VERSION: ${{ steps.mike-version.outputs.version }} + TITLE: ${{ steps.mike-version.outputs.title }} + ALIASES: ${{ steps.mike-version.outputs.aliases }} + # This is not ideal, we need to define all these variables here + # because we need to calculate all the repository version information + # to be able to show the correct versions in the documentation when + # building it. + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPO: ${{ github.repository }} + GIT_REF: ${{ github.ref }} + GIT_SHA: ${{ github.sha }} + run: | + mike deploy --update-aliases --title "$TITLE" "$VERSION" $ALIASES + + - name: Sort site versions + if: steps.mike-version.outputs.version + run: | + git checkout gh-pages + python -m frequenz.repo.config.cli.version.mike.sort versions.json + git commit -a -m "Sort versions.json" + + - name: Publish site + if: steps.mike-version.outputs.version run: | - mike deploy --push --update-aliases "$VERSION" $ALIASES + git push origin gh-pages create-github-release: name: Create GitHub release @@ -263,7 +302,7 @@ jobs: # discussions to create the release announcement in the discussion forums contents: write discussions: write - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - name: Download distribution files uses: actions/download-artifact@v4 @@ -305,7 +344,7 @@ jobs: publish-to-pypi: name: Publish packages to PyPI needs: ["create-github-release"] - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 permissions: # For trusted publishing. See: # https://blog.pypi.org/posts/2023-04-20-introducing-trusted-publishers/ diff --git a/pyproject.toml b/pyproject.toml index 30271100..4ae660c7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ requires = [ "setuptools == 75.8.0", "setuptools_scm[toml] == 8.1.0", - "frequenz-repo-config[api] == 0.11.0", + "frequenz-repo-config[api] == 0.12.3", # We need to pin the protobuf, grpcio and grpcio-tools dependencies to make # sure the code is generated using the minimum supported versions, as older # versions can't work with code that was generated with newer versions. @@ -62,8 +62,8 @@ dev-mkdocs = [ "mkdocs-gen-files == 0.5.0", "mkdocs-literate-nav == 0.6.1", "mkdocs-material == 9.6.1", - "mkdocstrings[python] == 0.27.0", - "frequenz-repo-config[api] == 0.11.0", + "mkdocstrings[python] == 0.28.3", + "frequenz-repo-config[api] == 0.12.3", ] dev-mypy = [ "mypy == 1.14.1", @@ -71,7 +71,7 @@ dev-mypy = [ # For checking the noxfile, docs/ script, and tests "frequenz-api-common[dev-mkdocs,dev-noxfile,dev-pytest]", ] -dev-noxfile = ["nox == 2025.2.9", "frequenz-repo-config[api] == 0.11.0"] +dev-noxfile = ["nox == 2025.2.9", "frequenz-repo-config[api] == 0.12.3"] dev-pylint = [ "pylint == 3.3.4", # For checking the noxfile, docs/ script, and tests @@ -79,7 +79,7 @@ dev-pylint = [ ] dev-pytest = [ "pytest == 8.3.4", - "frequenz-repo-config[extra-lint-examples] == 0.11.0", + "frequenz-repo-config[extra-lint-examples] == 0.12.3", ] dev = [ "frequenz-api-common[dev-mkdocs,dev-flake8,dev-formatting,dev-mkdocs,dev-mypy,dev-noxfile,dev-pylint,dev-pytest]", @@ -140,6 +140,7 @@ disable = [ [tool.pytest.ini_options] testpaths = ["pytests"] +addopts = "-W=all -Werror -Wdefault::DeprecationWarning -Wdefault::PendingDeprecationWarning -vv" [tool.mypy] explicit_package_bases = true