Skip to content

Commit 9a7bbe9

Browse files
llucaxllucax
authored
Several fixes and improvements (#141)
These fixes and improvements are mostly *imported* from the [SDK](https://github.com/frequenz-floss/frequenz-sdk-python/). - mkdocs: Add a few markdown extensions - Fix mermaid diagrams rendering - Show inherited members in the API reference - Add mkdocs-macros plugin - Move `mkdocstrings_autoapi.py` to `docs/_scripts` - mkdocs: Don't watch for the README - Make code annotations numbered - cookiecutter: Update the SDK dependency - Bump dependencies - docs: Move all support files to directories starting with `_` - Add `mypy` overrides for API repos too
2 parents 34313c3 + 25258eb commit 9a7bbe9

File tree

93 files changed

+2648
-210
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

93 files changed

+2648
-210
lines changed

‎.github/containers/nox-cross-arch/arm64-ubuntu-20.04-python-3.11.Dockerfile‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
# License: MIT
2+
# Copyright © 2023 Frequenz Energy-as-a-Service GmbH
3+
4+
# This Dockerfile is used to test the installation of the python package in
5+
# multiple platforms in the CI. It is not used to build the package itself.
6+
7+
FROM docker.io/library/ubuntu:20.04
8+
9+
ENV DEBIAN_FRONTEND=noninteractive
10+
11+
# Install Python 3.11 and curl to install pip later
12+
RUN apt-get update -y && \
13+
apt-get install --no-install-recommends -y \
14+
software-properties-common && \
15+
add-apt-repository ppa:deadsnakes/ppa && \
16+
apt-get install --no-install-recommends -y \
17+
ca-certificates \
18+
curl \
19+
git \
20+
python3.11 \
21+
python3.11-distutils && \
22+
apt-get clean && \
23+
rm -rf /var/lib/apt/lists/*
24+
25+
# Install pip
26+
RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11
27+
28+
RUN update-alternatives --install \
29+
/usr/local/bin/python python /usr/bin/python3.11 1 && \
30+
python -m pip install --upgrade --no-cache-dir pip
31+
32+
COPY entrypoint.bash /usr/bin/entrypoint.bash
33+
34+
ENTRYPOINT ["/usr/bin/entrypoint.bash"]

‎.github/containers/nox-cross-arch/entrypoint.bash‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/bin/bash
2+
# License: MIT
3+
# Copyright © 2023 Frequenz Energy-as-a-Service GmbH
4+
set -e
5+
6+
echo "System details:" $(uname -a)
7+
echo "Machine:" $(uname -m)
8+
9+
exec "$@"

‎.github/containers/test-installation/Dockerfile‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
# License: MIT
2+
# Copyright © 2023 Frequenz Energy-as-a-Service GmbH
3+
4+
# This Dockerfile is used to test the installation of the python package in
5+
# multiple platforms in the CI. It is not used to build the package itself.
6+
7+
FROM --platform=${TARGETPLATFORM} python:3.11-slim
8+
9+
RUN python -m pip install --upgrade --no-cache-dir pip
10+
11+
COPY dist dist
12+
RUN pip install dist/*.whl && \
13+
rm -rf dist

‎.github/workflows/ci.yaml‎

Lines changed: 155 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ jobs:
4444
run: env
4545

4646
- name: Fetch sources
47-
uses: actions/checkout@v3
47+
uses: actions/checkout@v4
4848
with:
4949
submodules: true
5050

@@ -79,12 +79,137 @@ jobs:
7979
run: nox -R -e "$NOX_SESSION"
8080
timeout-minutes: 60
8181

82+
# This job runs if all the `nox` matrix jobs ran and succeeded.
83+
# It is only used to have a single job that we can require in branch
84+
# protection rules, so we don't have to update the protection rules each time
85+
# we add or remove a job from the matrix.
86+
nox-all:
87+
# The job name should match the name of the `nox` job.
88+
name: Test with nox
89+
needs: ["nox"]
90+
runs-on: ubuntu-20.04
91+
steps:
92+
- name: Return true
93+
run: "true"
94+
95+
nox-cross-arch:
96+
name: Cross-arch tests with nox
97+
if: github.event_name != 'pull_request'
98+
strategy:
99+
fail-fast: false
100+
# Before adding new items to this matrix, make sure that a dockerfile
101+
# exists for the combination of items in the matrix.
102+
# Refer to .github/containers/nox-cross-arch/README.md to learn how to
103+
# add and name new dockerfiles.
104+
matrix:
105+
arch:
106+
- arm64
107+
os:
108+
- ubuntu-20.04
109+
python:
110+
- "3.11"
111+
nox-session:
112+
- "pytest_min"
113+
- "pytest_max"
114+
runs-on: ${{ matrix.os }}
115+
116+
steps:
117+
- name: Fetch sources
118+
uses: actions/checkout@v4
119+
120+
- name: Set up QEMU
121+
uses: docker/setup-qemu-action@v3
122+
with:
123+
platforms: linux/${{ matrix.arch }}
124+
125+
- name: Set up Docker Buildx
126+
uses: docker/setup-buildx-action@v3
127+
128+
# This is a workaround to prevent the cache from growing indefinitely.
129+
# https://docs.docker.com/build/ci/github-actions/cache/#local-cache
130+
# https://github.com/docker/build-push-action/issues/252
131+
# https://github.com/moby/buildkit/issues/1896
132+
- name: Cache container layers
133+
uses: actions/cache@v3
134+
with:
135+
path: /tmp/.buildx-cache
136+
key: ${{ runner.os }}-buildx-nox-${{ matrix.arch }}-${{ matrix.os }}-${{ matrix.python }}
137+
138+
- name: Build image
139+
uses: docker/build-push-action@v5
140+
with:
141+
context: .github/containers/nox-cross-arch
142+
file: .github/containers/nox-cross-arch/${{ matrix.arch }}-${{ matrix.os }}-python-${{ matrix.python }}.Dockerfile
143+
platforms: linux/${{ matrix.arch }}
144+
tags: localhost/nox-cross-arch:latest
145+
push: false
146+
load: true
147+
cache-from: type=local,src=/tmp/.buildx-cache
148+
cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
149+
150+
# Refer to the workaround mentioned above
151+
- name: Move cache
152+
run: |
153+
rm -rf /tmp/.buildx-cache
154+
mv /tmp/.buildx-cache-new /tmp/.buildx-cache
155+
156+
# Cache pip downloads
157+
- name: Cache pip downloads
158+
uses: actions/cache@v3
159+
with:
160+
path: /tmp/pip-cache
161+
key: nox-${{ matrix.nox-session }}-${{ matrix.arch }}-${{ matrix.os }}-${{ matrix.python }}-${{ hashFiles('**/pyproject.toml') }}
162+
163+
# This ensures that the docker container has access to the pip cache.
164+
# Changing the user in the docker-run step causes it to fail due to
165+
# incorrect permissions. Setting the ownership of the pip cache to root
166+
# before running is a workaround to this issue.
167+
- name: Set pip cache owners to root for docker
168+
run: if [[ -e /tmp/pip-cache ]]; then sudo chown -R root:root /tmp/pip-cache; fi
169+
170+
- name: Run nox
171+
run: |
172+
docker run \
173+
--rm \
174+
-v $(pwd):/${{ github.workspace }} \
175+
-v /tmp/pip-cache:/root/.cache/pip \
176+
-w ${{ github.workspace }} \
177+
--net=host \
178+
--platform linux/${{ matrix.arch }} \
179+
localhost/nox-cross-arch:latest \
180+
bash -c "pip install -e .[dev-noxfile]; nox --install-only -e ${{ matrix.nox-session }}; pip freeze; nox -e ${{ matrix.nox-session }}"
181+
timeout-minutes: 30
182+
183+
# This ensures that the runner has access to the pip cache.
184+
- name: Reset pip cache ownership
185+
if: always()
186+
run: sudo chown -R $USER:$USER /tmp/pip-cache
187+
188+
# This job runs if all the `nox-cross-arch` matrix jobs ran and succeeded.
189+
# As the `nox-all` job, its main purpose is to provide a single point of
190+
# reference in branch protection rules, similar to how `nox-all` operates.
191+
# However, there's a crucial difference: the `nox-cross-arch` job is omitted
192+
# in PRs. Without the `nox-cross-arch-all` job, the inner matrix wouldn't be
193+
# expanded in such scenarios. This would lead to the CI indefinitely waiting
194+
# for these jobs to complete due to the branch protection rules, essentially
195+
# causing it to hang. This behavior is tied to a recognized GitHub matrices
196+
# issue when certain jobs are skipped. For a deeper understanding, refer to:
197+
# https://github.com/orgs/community/discussions/9141
198+
nox-cross-arch-all:
199+
# The job name should match the name of the `nox-cross-arch` job.
200+
name: Cross-arch tests with nox
201+
needs: ["nox-cross-arch"]
202+
runs-on: ubuntu-20.04
203+
steps:
204+
- name: Return true
205+
run: "true"
206+
82207
build:
83208
name: Build distribution packages
84209
runs-on: ubuntu-20.04
85210
steps:
86211
- name: Fetch sources
87-
uses: actions/checkout@v3
212+
uses: actions/checkout@v4
88213
with:
89214
submodules: true
90215

@@ -110,13 +235,38 @@ jobs:
110235
path: dist/
111236
if-no-files-found: error
112237

238+
test-installation:
239+
name: Test package installation in different architectures
240+
needs: ["build"]
241+
runs-on: ubuntu-20.04
242+
steps:
243+
- name: Fetch sources
244+
uses: actions/checkout@v4
245+
- name: Download package
246+
uses: actions/download-artifact@v3
247+
with:
248+
name: dist-packages
249+
path: dist
250+
- name: Set up QEMU
251+
uses: docker/setup-qemu-action@v3
252+
- name: Set up docker-buildx
253+
uses: docker/setup-buildx-action@v3
254+
- name: Test Installation
255+
uses: docker/build-push-action@v5
256+
with:
257+
context: .
258+
file: .github/containers/test-installation/Dockerfile
259+
platforms: linux/amd64,linux/arm64
260+
tags: localhost/test-installation
261+
push: false
262+
113263
test-docs:
114264
name: Test documentation website generation
115265
if: github.event_name != 'push'
116266
runs-on: ubuntu-20.04
117267
steps:
118268
- name: Fetch sources
119-
uses: actions/checkout@v3
269+
uses: actions/checkout@v4
120270
with:
121271
submodules: true
122272

@@ -151,7 +301,7 @@ jobs:
151301

152302
publish-docs:
153303
name: Publish documentation website to GitHub pages
154-
needs: ["nox", "build"]
304+
needs: ["nox-all", "nox-cross-arch-all", "test-installation"]
155305
if: github.event_name == 'push'
156306
runs-on: ubuntu-20.04
157307
permissions:
@@ -196,7 +346,7 @@ jobs:
196346
197347
- name: Fetch sources
198348
if: steps.mike-metadata.outputs.version
199-
uses: actions/checkout@v3
349+
uses: actions/checkout@v4
200350
with:
201351
submodules: true
202352

‎.github/workflows/labeler.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ jobs:
1818
# only use hashes to pick the action to execute (instead of tags or branches).
1919
# For more details read:
2020
# https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
21-
uses: actions/labeler@0776a679364a9a16110aac8d0f40f5e11009e327 # 4.0.4
21+
uses: actions/labeler@ac9175f8a1f3625fd0d4fb234536d26811351594 # 4.3.0
2222
with:
2323
repo-token: "${{ secrets.GITHUB_TOKEN }}"
2424
dot: true

‎README.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,14 @@ It offers:
1616
common checks.
1717
* Tools to build protobuf/grpc files as Python, including type information.
1818

19+
## Supported Platforms
20+
21+
The following platforms are officially supported (test):
22+
23+
- **Python:** 3.11
24+
- **Operating System:** Ubuntu Linux 20.04
25+
- **Architectures:** amd64, arm64
26+
1927
## Quick Example
2028

2129
To start a new project, you should first [install

‎RELEASE_NOTES.md‎

Lines changed: 31 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,20 +10,48 @@
1010

1111
### Cookiecutter template
1212

13-
<!-- Here upgrade steps for cookiecutter specifically -->
13+
- `mkdocs`
14+
15+
- The script `docs/mkdocstrings_autoapi.py` was moved to `docs/_scripts/mkdocstrings.py`.
16+
- Note that now code annotations will be numbered. This is useful to hint about the order one should read the annotations.
17+
- The following files were renamed to keep the documentation directory clean for documentation files:
18+
19+
- `docs/css` -> `docs/_css`
20+
- `docs/overrides` -> `docs/_overrides`
21+
- `logo.png` -> `docs/_img/logo.png`
22+
23+
- CI
24+
25+
- You can now make your branch protection rule only require the "Test with nox" CI job to pass. All the matrix expansions will merge into it, so there is no need to change branch protection rules if matrix elements are added or removed.
1426

1527
## New Features
1628

1729
<!-- Here goes the main new features and examples or instructions on how to use them -->
1830

1931
### Cookiecutter template
2032

21-
<!-- Here new features for cookiecutter specifically -->
33+
- `mkdocs`
34+
35+
- New markdown extensions: [`def_list` / `task_list`](https://squidfunk.github.io/mkdocs-material/reference/lists/) and [`footnotes`](https://squidfunk.github.io/mkdocs-material/reference/footnotes/).
36+
- New [`mkdocs-macros`](https://mkdocs-macros-plugin.readthedocs.io/en/latest/) extension.
37+
- Show inherited attributes in the documentation.
38+
- Make code annotations numbered. This is useful to hint about the order one should read the annotations.
39+
- Updated dependencies.
40+
41+
- CI
42+
43+
- Add CI job to test package installation on multiple platforms (amd64 and arm64).
44+
- Add CI job to run the tests in arm64.
45+
- Add a CI job to *join* all `nox` runs, so only one branch protection rule needs to be used.
2246

2347
## Bug Fixes
2448

2549
<!-- Here goes notable bug fixes that are worth a special mention or explanation -->
2650

2751
### Cookiecutter template
2852

29-
<!-- Here bug fixes for cookiecutter specifically -->
53+
- `mkdocs`
54+
55+
- Fixed mermaid diagrams not rendering in the documentation.
56+
- `mypy` ignores for `cookiecutter` have been removed. They should have never be there as generated projects don't use `cookiecutter`.
57+
- `mypy` overrides now are applied to API projects too.

‎cookiecutter/{{cookiecutter.github_repo_name}}/.github/containers/nox-cross-arch/arm64-ubuntu-20.04-python-3.11.Dockerfile‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
# License: {{cookiecutter.license}}
2+
# Copyright © {% now 'utc', '%Y' %} {{cookiecutter.author_name}}
3+
{% raw -%}
4+
5+
# This Dockerfile is used to run the tests in architectures not supported by
6+
# GitHub Actions.
7+
8+
FROM docker.io/library/ubuntu:20.04
9+
10+
ENV DEBIAN_FRONTEND=noninteractive
11+
12+
# Install Python 3.11 and curl to install pip later
13+
RUN apt-get update -y && \
14+
apt-get install --no-install-recommends -y \
15+
software-properties-common && \
16+
add-apt-repository ppa:deadsnakes/ppa && \
17+
apt-get install --no-install-recommends -y \
18+
ca-certificates \
19+
curl \
20+
git \
21+
python3.11 \
22+
python3.11-distutils && \
23+
apt-get clean && \
24+
rm -rf /var/lib/apt/lists/*
25+
26+
# Install pip
27+
RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11
28+
29+
RUN update-alternatives --install \
30+
/usr/local/bin/python python /usr/bin/python3.11 1 && \
31+
python -m pip install --upgrade --no-cache-dir pip
32+
33+
COPY entrypoint.bash /usr/bin/entrypoint.bash
34+
35+
ENTRYPOINT ["/usr/bin/entrypoint.bash"]
36+
{%- endraw %}

‎cookiecutter/{{cookiecutter.github_repo_name}}/.github/containers/nox-cross-arch/entrypoint.bash‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#!/bin/bash
2+
# License: {{cookiecutter.license}}
3+
# Copyright © {% now 'utc', '%Y' %} {{cookiecutter.author_name}}
4+
{% raw -%}
5+
set -e
6+
7+
echo "System details:" $(uname -a)
8+
echo "Machine:" $(uname -m)
9+
10+
exec "$@"
11+
{%- endraw %}

0 commit comments

Comments
 (0)