From 324e1b74d0c77e19f7f78ee37a2a38ead07f4831 Mon Sep 17 00:00:00 2001 From: Leandro Lucarella Date: Mon, 3 Jul 2023 13:09:23 +0200 Subject: [PATCH 1/4] Migrate to use repo-config Now this project uses repo-config: https://github.com/frequenz-floss/frequenz-repo-config-python The files were updated by generating the project using the cookiecutter template. Some dependencies needed to be upgraded as part of the process. Now all dependencies are pinned and kept at the pyproject.toml file. Signed-off-by: Leandro Lucarella --- .cookiecutter-replay.json | 28 ++++++++ .github/ISSUE_TEMPLATE/bug.yml | 8 ++- .github/ISSUE_TEMPLATE/feature.yml | 5 +- .github/RELEASE_NOTES.template.md | 4 +- .github/labeler.yml | 6 +- .github/workflows/ci.yaml | 97 ++++++++++++++----------- .github/workflows/labeler.yml | 17 +++-- .gitignore | 21 +++++- CONTRIBUTING.md | 64 +++++++++++++---- LICENSE | 2 +- README.md | 7 +- RELEASE_NOTES.md | 2 +- docs/SUMMARY.md | 2 +- docs/mkdocstrings_autoapi.py | 60 ++-------------- docs/overrides/main.html | 2 +- mkdocs.yml | 13 ++-- noxfile.py | 110 ++--------------------------- pyproject.toml | 85 ++++++++++++++++------ 18 files changed, 261 insertions(+), 272 deletions(-) create mode 100644 .cookiecutter-replay.json diff --git a/.cookiecutter-replay.json b/.cookiecutter-replay.json new file mode 100644 index 00000000..e04d5a1f --- /dev/null +++ b/.cookiecutter-replay.json @@ -0,0 +1,28 @@ +{ + "cookiecutter": { + "type": "lib", + "name": "channels", + "description": "Channel implementations for Python", + "title": "Frequenz channels", + "keywords": "channel", + "github_org": "frequenz-floss", + "license": "MIT", + "author_name": "Frequenz Energy-as-a-Service GmbH", + "author_email": "floss@frequenz.com", + "python_package": "frequenz.channels", + "pypi_package_name": "frequenz-channels", + "github_repo_name": "frequenz-channels-python", + "default_codeowners": "@frequenz-floss/python-sdk-team", + "_extensions": [ + "jinja2_time.TimeExtension", + "local_extensions.default_codeowners", + "local_extensions.github_repo_name", + "local_extensions.keywords", + "local_extensions.pypi_package_name", + "local_extensions.python_package", + "local_extensions.src_path", + "local_extensions.title" + ], + "_template": "gh:frequenz-floss/frequenz-repo-config-python" + } +} \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml index d9b39289..3b12aa28 100644 --- a/.github/ISSUE_TEMPLATE/bug.yml +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -5,7 +5,9 @@ name: Report something is not working properly 🐛 description: Use this if there is something that is not working properly. If you are not sure or you need help making something work, please ask a question instead. -labels: priority:❓, type:bug +labels: + - "priority:❓" + - "type:bug" body: - type: markdown attributes: @@ -59,6 +61,6 @@ body: label: Extra information description: Please write here any extra information you think it might be relevant, - e.g., if this didn't happened before, or if you suspect where the - problem might be. + e.g., if this didn't happen before, or if you suspect where the problem + might be. placeholder: Any extra information you think it might be relevant. diff --git a/.github/ISSUE_TEMPLATE/feature.yml b/.github/ISSUE_TEMPLATE/feature.yml index 061b837c..459dd4b1 100644 --- a/.github/ISSUE_TEMPLATE/feature.yml +++ b/.github/ISSUE_TEMPLATE/feature.yml @@ -3,7 +3,10 @@ name: Request a feature or enhancement ✨ description: Use this if something is missing or could be done better or more easily. -labels: part:❓, priority:❓, type:enhancement +labels: + - "part:❓" + - "priority:❓" + - "type:enhancement" body: - type: markdown attributes: diff --git a/.github/RELEASE_NOTES.template.md b/.github/RELEASE_NOTES.template.md index 5468717a..96a0240b 100644 --- a/.github/RELEASE_NOTES.template.md +++ b/.github/RELEASE_NOTES.template.md @@ -1,4 +1,4 @@ -# Frequenz Channels Release Notes +# Frequenz channels Release Notes ## Summary @@ -6,7 +6,7 @@ ## Upgrading - + ## New Features diff --git a/.github/labeler.yml b/.github/labeler.yml index 7bb5972b..6989df44 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -9,6 +9,7 @@ "part:docs": - "**/*.md" - "docs/**" + - "examples/**" - LICENSE "part:tests": @@ -18,14 +19,13 @@ - "**/*.ini" - "**/*.toml" - "**/*.yaml" - - "*requirements*.txt" + - "**/*.yml" - ".git*" - ".git*/**" + - docs/*.py - CODEOWNERS - MANIFEST.in - - docs/mkdocstrings_autoapi.py - noxfile.py - - setup.py "part:channels": - any: diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 71b849af..ba8647c4 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,9 +1,11 @@ -name: frequenz-channels-python +name: CI on: merge_group: pull_request: push: + # We need to explicitly include tags because otherwise when adding + # `branches-ignore` it will only trigger on branches. tags: - '*' branches-ignore: @@ -14,44 +16,47 @@ on: workflow_dispatch: env: - DEFAULT_PYTHON_VERSION: "3.11" + # Please make sure this version is included in the `matrix`, as the + # `matrix` section can't use `env`, so it must be entered manually + DEFAULT_PYTHON_VERSION: '3.11' + # It would be nice to be able to also define a DEFAULT_UBUNTU_VERSION + # but sadly `env` can't be used either in `runs-on`. jobs: - test: + nox: + name: Test with nox strategy: + fail-fast: false matrix: os: - ubuntu-20.04 - python-version: + python: - "3.11" runs-on: ${{ matrix.os }} steps: - - name: Fetch sources - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-${{ matrix.python-version }}-pip-${{ hashFiles('pyproject.toml') }} - restore-keys: | - ${{ runner.os }}-${{ matrix.python-version }}-pip- - - - name: Install required Python packages - run: | - python -m pip install --upgrade pip - python -m pip install nox - - - name: run nox - run: nox - timeout-minutes: 10 - - build-dist: + - name: Fetch sources + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + cache: 'pip' + + - name: Install required Python packages + run: | + python -m pip install --upgrade pip + python -m pip install -e .[dev-noxfile] + + - name: Run nox + # To speed things up a bit we use the speciall ci_checks_max session + # that uses the same venv to run multiple linting sessions + run: nox -e ci_checks_max pytest_min + timeout-minutes: 10 + + build: + name: Build distribution packages runs-on: ubuntu-20.04 steps: - name: Fetch sources @@ -61,8 +66,9 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ env.DEFAULT_PYTHON_VERSION }} + cache: 'pip' - - name: Install build dependencies + - name: Install required Python packages run: | python -m pip install -U pip python -m pip install -U build @@ -70,14 +76,15 @@ jobs: - name: Build the source and binary distribution run: python -m build - - name: Upload dist files + - name: Upload distribution files uses: actions/upload-artifact@v3 with: - name: frequenz-channels-python-dist + name: dist-packages path: dist/ if-no-files-found: error - test-generate-docs: + test-docs: + name: Test documentation website generation if: github.event_name != 'push' runs-on: ubuntu-20.04 steps: @@ -91,11 +98,12 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ env.DEFAULT_PYTHON_VERSION }} + cache: 'pip' - name: Install build dependencies run: | python -m pip install -U pip - python -m pip install .[docs] + python -m pip install .[dev-mkdocs] - name: Generate the documentation env: @@ -107,12 +115,13 @@ jobs: - name: Upload site uses: actions/upload-artifact@v3 with: - name: frequenz-channels-python-site + name: docs-site path: site/ if-no-files-found: error publish-docs: - needs: ["test", "build-dist"] + name: Publish documentation website to GitHub pages + needs: ["nox", "build"] if: github.event_name == 'push' runs-on: ubuntu-20.04 permissions: @@ -168,12 +177,13 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ env.DEFAULT_PYTHON_VERSION }} + cache: 'pip' - name: Install build dependencies if: steps.mike-metadata.outputs.version run: | python -m pip install -U pip - python -m pip install .[docs] + python -m pip install .[dev-mkdocs] - name: Fetch the gh-pages branch if: steps.mike-metadata.outputs.version @@ -188,9 +198,10 @@ jobs: mike deploy --push --update-aliases "$VERSION" $ALIASES create-github-release: + name: Create GitHub release needs: ["publish-docs"] # Create a release only on tags creation - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') permissions: # We need write permissions on contents to create GitHub releases and on # discussions to create the release announcement in the discussion forums @@ -198,10 +209,10 @@ jobs: discussions: write runs-on: ubuntu-20.04 steps: - - name: Download dist files + - name: Download distribution files uses: actions/download-artifact@v3 with: - name: frequenz-channels-python-dist + name: dist-packages path: dist - name: Download RELEASE_NOTES.md @@ -225,7 +236,6 @@ jobs: if echo "$REF_NAME" | grep -- -; then extra_opts=" --prerelease"; fi gh release create \ -R "$REPOSITORY" \ - --discussion-category announcements \ --notes-file RELEASE_NOTES.md \ --generate-notes \ $extra_opts \ @@ -237,6 +247,7 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} publish-to-pypi: + name: Publish packages to PyPI needs: ["create-github-release"] runs-on: ubuntu-20.04 permissions: @@ -244,10 +255,10 @@ jobs: # https://blog.pypi.org/posts/2023-04-20-introducing-trusted-publishers/ id-token: write steps: - - name: Download dist files + - name: Download distribution files uses: actions/download-artifact@v3 with: - name: frequenz-channels-python-dist + name: dist-packages path: dist - name: Publish the Python distribution to PyPI diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index eafafa8e..cf458e6b 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -1,13 +1,5 @@ name: Pull Request Labeler -# XXX: !!! SECURITY WARNING !!! -# pull_request_target has write access to the repo, and can read secrets. We -# need to audit any external actions executed in this workflow and make sure no -# checked out code is run (not even installing dependencies, as installing -# dependencies usually can execute pre/post-install scripts). We should also -# only use hashes to pick the action to execute (instead of tags or branches). -# For more details read: -# https://securitylab.github.com/research/github-actions-preventing-pwn-requests/ on: [pull_request_target] jobs: @@ -18,7 +10,14 @@ jobs: runs-on: ubuntu-latest steps: - name: Labeler - # Only use hashes, see the security comment above + # XXX: !!! SECURITY WARNING !!! + # pull_request_target has write access to the repo, and can read secrets. We + # need to audit any external actions executed in this workflow and make sure no + # checked out code is run (not even installing dependencies, as installing + # dependencies usually can execute pre/post-install scripts). We should also + # only use hashes to pick the action to execute (instead of tags or branches). + # For more details read: + # https://securitylab.github.com/research/github-actions-preventing-pwn-requests/ uses: actions/labeler@0967ca812e7fdc8f5f71402a1b486d5bd061fe20 # 4.2.0 with: repo-token: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.gitignore b/.gitignore index 2716aa17..6997f699 100644 --- a/.gitignore +++ b/.gitignore @@ -20,12 +20,12 @@ parts/ sdist/ var/ wheels/ -pip-wheel-metadata/ share/python-wheels/ *.egg-info/ .installed.cfg *.egg MANIFEST +.vscode # PyInstaller # Usually these files are written by a python script from a template @@ -39,6 +39,7 @@ pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ +.htmlcov*/ .tox/ .nox/ .coverage @@ -50,6 +51,7 @@ coverage.xml *.py,cover .hypothesis/ .pytest_cache/ +cover/ # Translations *.mo @@ -72,6 +74,7 @@ instance/ docs/_build/ # PyBuilder +.pybuilder/ target/ # Jupyter Notebook @@ -82,7 +85,9 @@ profile_default/ ipython_config.py # pyenv -.python-version +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version # pipenv # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. @@ -109,6 +114,9 @@ venv/ ENV/ env.bak/ venv.bak/ +# direnv https://github.com/direnv/direnv +.envrc +.direnv/ # Spyder project settings .spyderproject @@ -128,6 +136,15 @@ dmypy.json # Pyre type checker .pyre/ +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +.idea + # Automatically generated documentation docs/reference/ site/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b6ef7f31..81e98c42 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,5 +1,4 @@ -# Contributing to `frequenz-channels` - +# Contributing to Frequenz channels ## Build @@ -19,17 +18,55 @@ all the dependencies too): python -m pip install -e . ``` -You can also use `nox` to run the tests and other checks: +Or you can install all development dependencies (`mypy`, `pylint`, `pytest`, +etc.) in one go too: +```sh +python -m pip install -e .[dev] +``` + +If you don't want to install all the dependencies, you can also use `nox` to +run the tests and other checks creating its own virtual environments: ```sh -python -m pip install nox +python -m pip install .[dev-noxfile] nox ``` -To build the documentation, first install the dependencies: +You can also use `nox -R` to reuse the current testing environment to speed up +test at the expense of a higher chance to end up with a dirty test environment. + +### Running tests / checks individually + +For a better development test cycle you can install the runtime and test +dependencies and run `pytest` manually. + +```sh +python -m pip install .[dev-pytest] # included in .[dev] too + +# And for example +pytest tests/test_*.py +``` + +Or you can use `nox`: + +```sh +nox -R -s pytest -- test/test_*.py +``` + +The same appliest to `pylint` or `mypy` for example: + +```sh +nox -R -s pylint -- test/test_*.py +nox -R -s mypy -- test/test_*.py +``` + +### Building the documentation + +To build the documentation, first install the dependencies (if you didn't +install all `dev` dependencies): ```sh -python -m pip install -e .[docs] +python -m pip install -e .[dev-mkdocs] ``` Then you can build the documentation (it will be written in the `site/` @@ -85,9 +122,9 @@ These are the steps to create a new release: 1. Get the latest head you want to create a release from. 2. Update the `RELEASE_NOTES.md` file if it is not complete, up to date, and - clean from template comments (` ## Upgrading -* The minimum supported Python version was bumped to 3.11, downstream projects will need to upgrade too to use this version. - -* The `Select` class was replaced by a new `select()` function, with the following improvements: - - * Type-safe: proper type hinting by using the new helper type guard `selected_from()`. - * Fixes potential starvation issues. - * Simplifies the interface by providing values one-by-one. - * Guarantees there are no dangling tasks left behind when used as an async context manager. - - This new function is an [async iterator](https://docs.python.org/3.11/library/collections.abc.html#collections.abc.AsyncIterator), and makes sure no dangling tasks are left behind after a select loop is done. - - Example: - ```python - timer1 = Timer.periodic(datetime.timedelta(seconds=1)) - timer2 = Timer.timeout(datetime.timedelta(seconds=0.5)) - - async for selected in select(timer1, timer2): - if selected_from(selected, timer1): - # Beware: `selected.value` might raise an exception, you can always - # check for exceptions with `selected.exception` first or use - # a try-except block. You can also quickly check if the receiver was - # stopped and let any other unexpected exceptions bubble up. - if selected.was_stopped(): - print("timer1 was stopped") - continue - print(f"timer1: now={datetime.datetime.now()} drift={selected.value}") - timer2.stop() - elif selected_from(selected, timer2): - # Explicitly handling of exceptions - match selected.exception: - case ReceiverStoppedError(): - print("timer2 was stopped") - case Exception() as exception: - print(f"timer2: exception={exception}") - case None: - # All good, no exception, we can use `selected.value` safely - print( - f"timer2: now={datetime.datetime.now()} " - f"drift={selected.value}" - ) - case _ as unhanded: - assert_never(unhanded) - else: - # This is not necessary, as select() will check for exhaustiveness, but - # it is good practice to have it in case you forgot to handle a new - # receiver added to `select()` at a later point in time. - assert False - ``` + ## New Features -* A new `select()` function was added, please look at the *Upgrading* section for details. - -* A new `Event` utility receiver was added. - - This receiver can be made ready manually. It is mainly useful for testing but can also become handy in scenarios where a simple, on-off signal needs to be sent to a select loop for example. - - Example: - - ```python - import asyncio - from frequenz.channels import Receiver - from frequenz.channels.util import Event, select, selected_from - - other_receiver: Receiver[int] = ... - exit_event = Event() - - async def exit_after_10_seconds() -> None: - asyncio.sleep(10) - exit_event.set() - - asyncio.ensure_future(exit_after_10_seconds()) + - async for selected in select(exit_event, other_receiver): - if selected_from(selected, exit_event): - break - if selected_from(selected, other_receiver): - print(selected.value) - else: - assert False, "Unknow receiver selected" - ``` +## Bug Fixes -* The `Timer` class now has more descriptive `__str__` and `__repr__` methods. + From bf5333ab16de583a3e355f2d8cab7e883dc2e8eb Mon Sep 17 00:00:00 2001 From: Leandro Lucarella Date: Mon, 3 Jul 2023 13:30:05 +0200 Subject: [PATCH 4/4] Add missing dependency to repo-config for mkdocs When building the documentation with mkdocs we are actually using repo-config to generate the API documentation, so we need to add it to the dependencies. Signed-off-by: Leandro Lucarella --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 96585b1f..3a1102b8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,7 @@ dev-mkdocs = [ "mkdocs-material == 9.1.17", "mkdocs-section-index == 0.3.5", "mkdocstrings[python] == 0.22.0", + "frequenz-repo-config[lib] == 0.3.0", ] dev-mypy = [ "mypy == 1.2.0",