Skip to content

Commit b5b0f45

Browse files
Migrate to charmcraft 3 poetry plugin (#576)
data-platform-workflows and charmcraftcache are migrating from charmcraft 2 to 3 to enable 24.04 based charms & to enable use of the poetry plugin, which fixes several longstanding issues with charmcraft (e.g. canonical/charmcraft#1077)
1 parent d3aae57 commit b5b0f45

File tree

8 files changed

+109
-63
lines changed

8 files changed

+109
-63
lines changed

.github/workflows/ci.yaml

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ on:
1616
jobs:
1717
lint:
1818
name: Lint
19-
uses: canonical/data-platform-workflows/.github/workflows/lint.yaml@v24.0.6
19+
uses: canonical/data-platform-workflows/.github/workflows/lint.yaml@v26.0.0
2020

2121
unit-test:
2222
name: Unit test charm
@@ -56,9 +56,10 @@ jobs:
5656

5757
build:
5858
name: Build charm
59-
uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v24.0.6
59+
uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v26.0.0
6060
with:
6161
cache: true
62+
charmcraft-snap-channel: latest/beta/data-platform # TODO: remove after charmcraft 3.3 stable release
6263

6364
integration-test:
6465
strategy:
@@ -82,7 +83,7 @@ jobs:
8283
- lint
8384
- unit-test
8485
- build
85-
uses: canonical/data-platform-workflows/.github/workflows/integration_test_charm.yaml@v24.0.6
86+
uses: canonical/data-platform-workflows/.github/workflows/integration_test_charm.yaml@v26.0.0
8687
with:
8788
artifact-prefix: ${{ needs.build.outputs.artifact-prefix }}
8889
architecture: ${{ matrix.architecture }}

.github/workflows/release.yaml

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,15 +37,18 @@ jobs:
3737

3838
build:
3939
name: Build charm
40-
uses: canonical/data-platform-workflows/.github/workflows/[email protected]
40+
uses: canonical/data-platform-workflows/.github/workflows/[email protected]
41+
with:
42+
charmcraft-snap-channel: latest/beta/data-platform # TODO: remove after charmcraft 3.3 stable release
4143

4244
release:
4345
name: Release charm
4446
needs:
4547
- ci-tests
4648
- build
47-
uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@v24.0.6
49+
uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@v26.0.0
4850
with:
51+
charmcraft-snap-channel: latest/beta/data-platform # TODO: remove after charmcraft 3.3 stable release
4952
channel: 8.0/edge
5053
artifact-prefix: ${{ needs.build.outputs.artifact-prefix }}
5154
secrets:

.github/workflows/sync_docs.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ on:
1010
jobs:
1111
sync-docs:
1212
name: Sync docs from Discourse
13-
uses: canonical/data-platform-workflows/.github/workflows/sync_docs.yaml@v24.0.6
13+
uses: canonical/data-platform-workflows/.github/workflows/sync_docs.yaml@v26.0.0
1414
with:
1515
reviewers: a-velasco
1616
permissions:

charmcraft.yaml

Lines changed: 79 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -2,40 +2,89 @@
22
# See LICENSE file for licensing details.
33

44
type: charm
5-
bases:
6-
- name: ubuntu
7-
channel: "22.04"
8-
architectures: [amd64]
9-
- name: ubuntu
10-
channel: "22.04"
11-
architectures: [arm64]
5+
platforms:
6+
7+
8+
# Files implicitly created by charmcraft without a part:
9+
# - dispatch (https://github.com/canonical/charmcraft/pull/1898)
10+
# - manifest.yaml
11+
# (https://github.com/canonical/charmcraft/blob/9ff19c328e23b50cc06f04e8a5ad4835740badf4/charmcraft/services/package.py#L259)
12+
# Files implicitly copied/"primed" by charmcraft without a part:
13+
# - actions.yaml, config.yaml, metadata.yaml
14+
# (https://github.com/canonical/charmcraft/blob/9ff19c328e23b50cc06f04e8a5ad4835740badf4/charmcraft/services/package.py#L290-L293
15+
# https://github.com/canonical/charmcraft/blob/9ff19c328e23b50cc06f04e8a5ad4835740badf4/charmcraft/services/package.py#L156-L157)
1216
parts:
13-
files:
14-
plugin: dump
17+
# "poetry-deps" part name is a magic constant
18+
# https://github.com/canonical/craft-parts/pull/901
19+
poetry-deps:
20+
plugin: nil
21+
build-packages:
22+
- curl
23+
override-build: |
24+
# Use environment variable instead of `--break-system-packages` to avoid failing on older
25+
# versions of pip that do not recognize `--break-system-packages`
26+
# `--user` needed (in addition to `--break-system-packages`) for Ubuntu >=24.04
27+
PIP_BREAK_SYSTEM_PACKAGES=true python3 -m pip install --user --upgrade pip==24.3.1 # renovate: charmcraft-pip-latest
28+
29+
# Use uv to install poetry so that a newer version of Python can be installed if needed by poetry
30+
curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.5.15/uv-installer.sh | sh # renovate: charmcraft-uv-latest
31+
# poetry 2.0.0 requires Python >=3.9
32+
if ! "$HOME/.local/bin/uv" python find '>=3.9'
33+
then
34+
# Use first Python version that is >=3.9 and available in an Ubuntu LTS
35+
# (to reduce the number of Python versions we use)
36+
"$HOME/.local/bin/uv" python install 3.10.12 # renovate: charmcraft-python-ubuntu-22.04
37+
fi
38+
"$HOME/.local/bin/uv" tool install --no-python-downloads --python '>=3.9' poetry==2.0.0 --with poetry-plugin-export==1.8.0 # renovate: charmcraft-poetry-latest
39+
40+
ln -sf "$HOME/.local/bin/poetry" /usr/local/bin/poetry
41+
# "charm-poetry" part name is arbitrary; use for consistency
42+
# Avoid using "charm" part name since that has special meaning to charmcraft
43+
charm-poetry:
44+
# By default, the `poetry` plugin creates/primes these directories:
45+
# - lib, src
46+
# (https://github.com/canonical/charmcraft/blob/9ff19c328e23b50cc06f04e8a5ad4835740badf4/charmcraft/parts/plugins/_poetry.py#L76-L78)
47+
# - venv
48+
# (https://github.com/canonical/charmcraft/blob/9ff19c328e23b50cc06f04e8a5ad4835740badf4/charmcraft/parts/plugins/_poetry.py#L95
49+
# https://github.com/canonical/craft-parts/blob/afb0d652eb330b6aaad4f40fbd6e5357d358de47/craft_parts/plugins/base.py#L270)
50+
plugin: poetry
1551
source: .
16-
stage:
17-
# Exclude requirements.txt file during staging
18-
# Workaround for https://github.com/canonical/charmcraft/issues/1389 on charmcraft 2
19-
- -requirements.txt
20-
prime:
21-
- snap_revisions.json
22-
- scripts
23-
- templates
24-
charm:
25-
build-snaps:
26-
- rustup
52+
after:
53+
- poetry-deps
54+
poetry-export-extra-args: ['--only', 'main,charm-libs']
2755
build-packages:
28-
- libffi-dev
29-
- libssl-dev
30-
- pkg-config
56+
- libffi-dev # Needed to build Python dependencies with Rust from source
57+
- libssl-dev # Needed to build Python dependencies with Rust from source
58+
- pkg-config # Needed to build Python dependencies with Rust from source
3159
override-build: |
32-
rustup default stable
60+
# Workaround for https://github.com/canonical/charmcraft/issues/2068
61+
# rustup used to install rustc and cargo, which are needed to build Python dependencies with Rust from source
62+
if [[ "$CRAFT_PLATFORM" == [email protected]:* || "$CRAFT_PLATFORM" == [email protected]:* ]]
63+
then
64+
snap install rustup --classic
65+
else
66+
apt-get install rustup -y
67+
fi
68+
69+
# If Ubuntu version < 24.04, rustup was installed from snap instead of from the Ubuntu
70+
# archive—which means the rustup version could be updated at any time. Print rustup version
71+
# to build log to make changes to the snap's rustup version easier to track
72+
rustup --version
3373
34-
# Convert subset of poetry.lock to requirements.txt
35-
curl -sSL https://install.python-poetry.org | python3 -
36-
/root/.local/bin/poetry self add poetry-plugin-export
37-
/root/.local/bin/poetry export --only main,charm-libs --output requirements.txt
74+
# rpds-py (Python package) >=0.19.0 requires rustc >=1.76, which is not available in the
75+
# Ubuntu 22.04 archive. Install rustc and cargo using rustup instead of the Ubuntu archive
76+
rustup set profile minimal
77+
rustup default 1.83.0 # renovate: charmcraft-rust-latest
3878
3979
craftctl default
40-
charm-strict-dependencies: true
41-
charm-requirements: [requirements.txt]
80+
# Include requirements.txt in *.charm artifact for easier debugging
81+
cp requirements.txt "$CRAFT_PART_INSTALL/requirements.txt"
82+
# "files" part name is arbitrary; use for consistency
83+
files:
84+
plugin: dump
85+
source: .
86+
prime:
87+
- LICENSE
88+
- scripts
89+
- snap_revisions.json
90+
- templates

poetry.lock

Lines changed: 11 additions & 11 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

pyproject.toml

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -54,19 +54,19 @@ parameterized = "^0.9.0"
5454

5555
[tool.poetry.group.integration.dependencies]
5656
pytest = "^7.4.0"
57-
pytest-github-secrets = {git = "https://github.com/canonical/data-platform-workflows", tag = "v24.0.6", subdirectory = "python/pytest_plugins/github_secrets"}
58-
pytest-microceph = {git = "https://github.com/canonical/data-platform-workflows", tag = "v24.0.6", subdirectory = "python/pytest_plugins/microceph"}
57+
pytest-github-secrets = {git = "https://github.com/canonical/data-platform-workflows", tag = "v26.0.0", subdirectory = "python/pytest_plugins/github_secrets"}
58+
pytest-microceph = {git = "https://github.com/canonical/data-platform-workflows", tag = "v26.0.0", subdirectory = "python/pytest_plugins/microceph"}
5959
pytest-operator = "^0.28.0"
60-
pytest-operator-cache = {git = "https://github.com/canonical/data-platform-workflows", tag = "v24.0.6", subdirectory = "python/pytest_plugins/pytest_operator_cache"}
61-
pytest-operator-groups = {git = "https://github.com/canonical/data-platform-workflows", tag = "v24.0.6", subdirectory = "python/pytest_plugins/pytest_operator_groups"}
60+
pytest-operator-cache = {git = "https://github.com/canonical/data-platform-workflows", tag = "v26.0.0", subdirectory = "python/pytest_plugins/pytest_operator_cache"}
61+
pytest-operator-groups = {git = "https://github.com/canonical/data-platform-workflows", tag = "v26.0.0", subdirectory = "python/pytest_plugins/pytest_operator_groups"}
6262
juju = "^3.6.0.0"
6363
mysql-connector-python = "~8.0.33"
6464
tenacity = "^8.2.2"
6565
boto3 = "^1.28.11"
6666
pyyaml = "^6.0"
6767
urllib3 = "^1.26.16"
6868
allure-pytest = "^2.13.2"
69-
allure-pytest-collection-report = {git = "https://github.com/canonical/data-platform-workflows", tag = "v24.0.6", subdirectory = "python/pytest_plugins/allure_pytest_collection_report"}
69+
allure-pytest-collection-report = {git = "https://github.com/canonical/data-platform-workflows", tag = "v26.0.0", subdirectory = "python/pytest_plugins/allure_pytest_collection_report"}
7070

7171

7272
[tool.coverage.run]

tests/integration/helpers.py

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -982,15 +982,8 @@ def get_unit_by_index(app_name: str, units: list, index: int):
982982
return unit
983983

984984

985-
async def get_charm(charm_path: Union[str, Path], architecture: str, bases_index: int) -> Path:
985+
async def get_charm(charm_path: Union[str, Path], architecture: str) -> Path:
986986
"""Fetches packed charm from CI runner without checking for architecture."""
987987
charm_path = Path(charm_path)
988-
charmcraft_yaml = yaml.safe_load((charm_path / "charmcraft.yaml").read_text())
989-
assert charmcraft_yaml["type"] == "charm"
990-
991-
base = charmcraft_yaml["bases"][bases_index]
992-
build_on = base.get("build-on", [base])[0]
993-
version = build_on["channel"]
994-
packed_charms = list(charm_path.glob(f"*{version}-{architecture}.charm"))
995-
988+
packed_charms = list(charm_path.glob(f"*-{architecture}.charm"))
996989
return packed_charms[0].resolve(strict=True)

tests/integration/test_architecture.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
@markers.amd64_only
1616
async def test_arm_charm_on_amd_host(ops_test: OpsTest) -> None:
1717
"""Tries deploying an arm64 charm on amd64 host."""
18-
charm = await get_charm(".", "arm64", 1)
18+
charm = await get_charm(".", "arm64")
1919

2020
await ops_test.model.deploy(
2121
charm,
@@ -36,7 +36,7 @@ async def test_arm_charm_on_amd_host(ops_test: OpsTest) -> None:
3636
@markers.arm64_only
3737
async def test_amd_charm_on_arm_host(ops_test: OpsTest) -> None:
3838
"""Tries deploying an amd64 charm on arm64 host."""
39-
charm = await get_charm(".", "amd64", 0)
39+
charm = await get_charm(".", "amd64")
4040

4141
await ops_test.model.deploy(
4242
charm,

0 commit comments

Comments
 (0)