Skip to content

Commit 29bde17

Browse files
authored
Merge pull request #1859 from pi-hole/test_suite
Major overhaul to the test suite
2 parents f43429f + 1ae1414 commit 29bde17

File tree

11 files changed

+170
-207
lines changed

11 files changed

+170
-207
lines changed

.github/workflows/build-and-publish.yml

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,23 @@ env:
1616

1717
jobs:
1818
build:
19-
runs-on: ubuntu-latest
19+
runs-on: ${{ matrix.runner }}
2020
strategy:
2121
fail-fast: false
2222
matrix:
23-
platform: [linux/amd64, linux/386, linux/arm/v6, linux/arm/v7, linux/arm64, linux/riscv64]
23+
include:
24+
- platform: linux/amd64
25+
runner: ubuntu-latest
26+
- platform: linux/386
27+
runner: ubuntu-latest
28+
- platform: linux/arm/v6
29+
runner: ubuntu-24.04-arm
30+
- platform: linux/arm/v7
31+
runner: ubuntu-24.04-arm
32+
- platform: linux/arm64
33+
runner: ubuntu-24.04-arm
34+
- platform: linux/riscv64
35+
runner: ubuntu-24.04-arm
2436

2537
steps:
2638
- name: Prepare name for digest up/download

.github/workflows/build-and-test.yml

Lines changed: 33 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -3,30 +3,49 @@ on:
33
pull_request:
44

55
jobs:
6-
test:
7-
runs-on: ubuntu-latest
6+
build-and-test:
7+
runs-on: ${{ matrix.runner }}
88
strategy:
99
fail-fast: false
1010
matrix:
11-
# Official docker images for docker are only available for amd64 and arm64
12-
# TODO: Look at: https://github.com/docker-library/official-images#architectures-other-than-amd64
13-
# Is testing on all platforms really necessary?
14-
# Disabled arm64 tests for the time being, something is wrong with the test config and the volumes are getting shared between the test containers on different architectures
15-
#platform: [linux/amd64, linux/arm64]
16-
platform: [linux/amd64]
11+
include:
12+
- platform: linux/amd64
13+
runner: ubuntu-latest
14+
- platform: linux/386
15+
runner: ubuntu-latest
16+
- platform: linux/arm/v6
17+
runner: ubuntu-24.04-arm
18+
- platform: linux/arm/v7
19+
runner: ubuntu-24.04-arm
20+
- platform: linux/arm64
21+
runner: ubuntu-24.04-arm
22+
- platform: linux/riscv64
23+
runner: ubuntu-24.04-arm
24+
env:
25+
CI_ARCH: ${{ matrix.platform }}
1726
steps:
1827
- name: Checkout Repo
1928
uses: actions/checkout@v4
2029

2130
- name: Set up QEMU
2231
uses: docker/setup-qemu-action@v3
32+
33+
- name: Set up Python
34+
uses: actions/[email protected]
2335
with:
24-
platforms: ${{ matrix.platform }}
36+
python-version: "3.13"
37+
38+
- name: Run black formatter
39+
run: |
40+
pip install black
41+
black --check --diff test/tests/
42+
43+
- name: Install wheel
44+
run: pip install wheel
2545

26-
- name: Set up Docker Buildx
27-
uses: docker/setup-buildx-action@v3
46+
- name: Install dependencies
47+
run: pip install -r test/requirements.txt
2848

29-
- name: Run Tests
49+
- name: Test with tox
3050
run: |
31-
echo "Building image to test"
32-
PLATFORM=${{ matrix.platform }} ./build-and-test.sh
51+
CIPLATFORM=${{ env.CI_ARCH }} tox -c test/tox.ini

build-and-test.sh

Lines changed: 0 additions & 24 deletions
This file was deleted.

test/Dockerfile

Lines changed: 0 additions & 22 deletions
This file was deleted.

test/TESTING.md

Lines changed: 3 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,12 @@
11
# Prerequisites
22

3-
Make sure you have bash & docker installed.
4-
Python and some test hacks are crammed into the `Dockerfile_build` file for now.
5-
Revisions in the future may re-enable running python on your host (not just in docker).
3+
Make sure you have `docker`, `python` and `tox` installed.
64

75
# Running tests locally
86

9-
`./build-and-test.sh`
7+
`tox -c test/tox.ini`
108

119
Should result in:
1210

13-
- An image named `pihole:[branch-name]` being built
11+
- An image named `pihole:CI_container` being built
1412
- Tests being ran to confirm the image doesn't have any regressions
15-
16-
# Modify Pipfile
17-
18-
You can enter into the test docker image using `./build-and-test.sh enter`.
19-
From there, you can `cd test` and execute any needed pipenv commands.

test/cmd.sh

Lines changed: 0 additions & 11 deletions
This file was deleted.

test/requirements.txt

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
pytest == 8.4.1
2-
pytest-xdist == 3.7.0
32
pytest-testinfra == 10.2.2
4-
black == 25.1.0
53
pytest-clarity == 1.0.1
4+
tox == 4.27.0
5+
# Not adding pytest-xdist as using pytest with n > 1 cores
6+
# causes random issues with the emulated architectures

test/tests/conftest.py

Lines changed: 46 additions & 113 deletions
Original file line numberDiff line numberDiff line change
@@ -1,131 +1,64 @@
1-
import os
21
import pytest
32
import subprocess
43
import testinfra
5-
6-
local_host = testinfra.get_host("local://")
7-
check_output = local_host.check_output
8-
9-
TAIL_DEV_NULL = "tail -f /dev/null"
4+
import testinfra.backend.docker
5+
import os
106

117

12-
@pytest.fixture()
13-
def run_and_stream_command_output():
14-
def run_and_stream_command_output_inner(command, verbose=False):
15-
print("Running", command)
16-
build_env = os.environ.copy()
17-
build_env["PIHOLE_DOCKER_TAG"] = version
18-
build_result = subprocess.Popen(
19-
command.split(),
20-
env=build_env,
21-
stdout=subprocess.PIPE,
22-
stderr=subprocess.STDOUT,
23-
bufsize=1,
24-
universal_newlines=True,
8+
# Monkeypatch sh to bash, if they ever support non hard code /bin/sh this can go away
9+
# https://github.com/pytest-dev/pytest-testinfra/blob/master/testinfra/backend/docker.py
10+
def run_bash(self, command, *args, **kwargs):
11+
cmd = self.get_command(command, *args)
12+
if self.user is not None:
13+
out = self.run_local(
14+
"docker exec -u %s %s /bin/bash -c %s", self.user, self.name, cmd
2515
)
26-
if verbose:
27-
while build_result.poll() is None:
28-
for line in build_result.stdout:
29-
print(line, end="")
30-
build_result.wait()
31-
if build_result.returncode != 0:
32-
print(f" [i] Error running: {command}")
33-
print(build_result.stderr)
34-
35-
return run_and_stream_command_output_inner
36-
37-
38-
@pytest.fixture()
39-
def args_env():
40-
return '-e TZ="Europe/London" -e FTLCONF_dns_upstreams="8.8.8.8"'
41-
42-
43-
@pytest.fixture()
44-
def args(args_env):
45-
return "{}".format(args_env)
46-
47-
48-
@pytest.fixture()
49-
def test_args():
50-
"""test override fixture to provide arguments separate from our core args"""
51-
return ""
52-
53-
54-
def docker_generic(request, _test_args, _args, _image, _cmd, _entrypoint):
55-
# assert 'docker' in check_output('id'), "Are you in the docker group?"
56-
# Always appended PYTEST arg to tell pihole we're testing
57-
if "pihole" in _image and "PYTEST=1" not in _args:
58-
_args = "{} -e PYTEST=1".format(_args)
59-
docker_run = "docker run -d -t {args} {test_args} {entry} {image} {cmd}".format(
60-
args=_args, test_args=_test_args, entry=_entrypoint, image=_image, cmd=_cmd
61-
)
62-
# Print a human runable version of the container run command for faster debugging
63-
print(docker_run.replace("-d -t", "--rm -it").replace(TAIL_DEV_NULL, "bash"))
64-
docker_id = check_output(docker_run)
65-
66-
def teardown():
67-
check_output("docker logs {}".format(docker_id))
68-
check_output("docker rm -f {}".format(docker_id))
69-
70-
request.addfinalizer(teardown)
71-
docker_container = testinfra.backend.get_backend(
72-
"docker://" + docker_id, sudo=False
73-
)
74-
docker_container.id = docker_id
75-
76-
return docker_container
77-
78-
79-
@pytest.fixture
80-
def docker(request, test_args, args, image, cmd, entrypoint):
81-
"""One-off Docker container run"""
82-
return docker_generic(request, test_args, args, image, cmd, entrypoint)
83-
84-
85-
@pytest.fixture
86-
def entrypoint():
87-
return ""
16+
else:
17+
out = self.run_local("docker exec %s /bin/bash -c %s", self.name, cmd)
18+
out.command = self.encode(cmd)
19+
return out
8820

8921

90-
@pytest.fixture()
91-
def version():
92-
return os.environ.get("GIT_TAG", None)
22+
testinfra.backend.docker.DockerBackend.run = run_bash
9323

9424

95-
@pytest.fixture()
96-
def tag(version):
97-
return "{}".format(version)
25+
# scope='session' uses the same container for all the tests;
26+
# scope='function' uses a new container per test function.
27+
@pytest.fixture(scope="function")
28+
def docker(request):
29+
# Get platform from environment variable, default to None if not set
30+
platform = os.environ.get("CIPLATFORM")
9831

32+
# build the docker run command with args
33+
cmd = ["docker", "run", "-d", "-t"]
9934

100-
@pytest.fixture()
101-
def image(tag):
102-
image = "pihole"
103-
return "{}:{}".format(image, tag)
35+
# Only add platform flag if CIPLATFORM is set
36+
if platform:
37+
cmd.extend(["--platform", platform])
10438

39+
# Get env vars from parameterization
40+
env_vars = getattr(request, "param", [])
41+
if isinstance(env_vars, str):
42+
env_vars = [env_vars]
10543

106-
@pytest.fixture()
107-
def cmd():
108-
return TAIL_DEV_NULL
44+
# add parameterized environment variables
45+
for env_var in env_vars:
46+
cmd.extend(["-e", env_var])
10947

48+
# ensure PYTEST=1 is set
49+
if not any("PYTEST=1" in arg for arg in cmd):
50+
cmd.extend(["-e", "PYTEST=1"])
11051

111-
@pytest.fixture
112-
def slow():
113-
"""
114-
Run a slow check, check if the state is correct for `timeout` seconds.
115-
"""
116-
import time
52+
# add default TZ if not already set
53+
if not any("TZ=" in arg for arg in cmd):
54+
cmd.extend(["-e", 'TZ="Europe/London"'])
11755

118-
def _slow(check, timeout=20):
119-
timeout_at = time.time() + timeout
120-
while True:
121-
try:
122-
assert check()
123-
except AssertionError as e:
124-
if time.time() < timeout_at:
125-
time.sleep(1)
126-
else:
127-
raise e
128-
else:
129-
return
56+
# add the image name
57+
cmd.append("pihole:CI_container")
13058

131-
return _slow
59+
# run a container
60+
docker_id = subprocess.check_output(cmd).decode().strip()
61+
# return a testinfra connection to the container
62+
yield testinfra.get_host("docker://" + docker_id)
63+
# at the end of the test suite, destroy the container
64+
subprocess.check_call(["docker", "rm", "-f", docker_id])

0 commit comments

Comments
 (0)