From 5769312c14e8a7bf753ecfb64418928eec701259 Mon Sep 17 00:00:00 2001 From: Yvan Sraka Date: Tue, 12 Aug 2025 12:19:14 +0200 Subject: [PATCH 1/3] Install envoy using system manager --- nix/packages/default.nix | 1 + nix/packages/docker-ubuntu.nix | 7 ++++++- nix/packages/envoy-bin.nix | 30 +++++++++++++++++++++++++++ nix/systemModules/default.nix | 4 +++- nix/systemModules/envoy.nix | 23 ++++++++++++++++++++ nix/systemModules/tests/default.nix | 2 ++ nix/systemModules/tests/test_envoy.py | 3 +++ 7 files changed, 68 insertions(+), 2 deletions(-) create mode 100644 nix/packages/envoy-bin.nix create mode 100644 nix/systemModules/envoy.nix create mode 100644 nix/systemModules/tests/test_envoy.py diff --git a/nix/packages/default.nix b/nix/packages/default.nix index 980b3935e..ec8138086 100644 --- a/nix/packages/default.nix +++ b/nix/packages/default.nix @@ -36,6 +36,7 @@ dbmate-tool = pkgs.callPackage ./dbmate-tool.nix { inherit (self.supabase) defaults; }; docker-image-ubuntu = pkgs.callPackage ./docker-ubuntu.nix { }; docs = pkgs.callPackage ./docs.nix { }; + envoy-bin = pkgs.callPackage ./envoy-bin.nix { }; supabase-groonga = pkgs.callPackage ./groonga { }; local-infra-bootstrap = pkgs.callPackage ./local-infra-bootstrap.nix { }; migrate-tool = pkgs.callPackage ./migrate-tool.nix { psql_15 = self'.packages."psql_15/bin"; }; diff --git a/nix/packages/docker-ubuntu.nix b/nix/packages/docker-ubuntu.nix index ca77bf085..838f68f68 100644 --- a/nix/packages/docker-ubuntu.nix +++ b/nix/packages/docker-ubuntu.nix @@ -1,4 +1,9 @@ -{ runCommand, dockerTools, xz, buildEnv }: +{ + runCommand, + dockerTools, + xz, + buildEnv, +}: let ubuntu-cloudimg = let diff --git a/nix/packages/envoy-bin.nix b/nix/packages/envoy-bin.nix new file mode 100644 index 000000000..5e47787b8 --- /dev/null +++ b/nix/packages/envoy-bin.nix @@ -0,0 +1,30 @@ +{ + envoy-bin, + fetchurl, + stdenv, + ... +}: +let + version = "1.28.0"; + inherit (stdenv.hostPlatform) system; + throwSystem = throw "envoy-bin is not available for ${system}."; + plat = + { + aarch64-linux = "aarch_64"; + x86_64-linux = "x86_64"; + } + .${system} or throwSystem; + hash = + { + aarch64-linux = ""; + x86_64-linux = "sha256-JjlWPOm8CbHua9RzF2C1lsjtHkdM3YPMnfk2RRbhQ2c="; + } + .${system} or throwSystem; +in +envoy-bin.overrideAttrs { + inherit version; + src = fetchurl { + url = "https://github.com/envoyproxy/envoy/releases/download/v${version}/envoy-${version}-linux-${plat}"; + inherit hash; + }; +} diff --git a/nix/systemModules/default.nix b/nix/systemModules/default.nix index 14b459255..dfd30e666 100644 --- a/nix/systemModules/default.nix +++ b/nix/systemModules/default.nix @@ -4,6 +4,8 @@ { imports = [ ./tests ]; flake = { - systemModules = { }; + systemModules = { + envoy = ./envoy.nix; + }; }; } diff --git a/nix/systemModules/envoy.nix b/nix/systemModules/envoy.nix new file mode 100644 index 000000000..457b2640e --- /dev/null +++ b/nix/systemModules/envoy.nix @@ -0,0 +1,23 @@ +{ + lib, + nixosModulesPath, + self', + ... +}: +{ + imports = map (path: nixosModulesPath + path) [ + "/services/networking/envoy.nix" + ]; + config = { + services.envoy = { + enable = true; + package = self'.packages.envoy-bin; + # TODO: settings from postgres/ansible/files/envoy_config/ + }; + systemd.services.envoy = { + wantedBy = lib.mkForce [ + "system-manager.target" + ]; + }; + }; +} diff --git a/nix/systemModules/tests/default.nix b/nix/systemModules/tests/default.nix index cb489b714..99ced0c6d 100644 --- a/nix/systemModules/tests/default.nix +++ b/nix/systemModules/tests/default.nix @@ -14,11 +14,13 @@ lib = pkgs.lib; systemManagerConfig = self.inputs.system-manager.lib.makeSystemConfig { modules = [ + self.systemModules.envoy ({ services.nginx.enable = true; nixpkgs.hostPlatform = pkgs.system; }) ]; + extraSpecialArgs = { inherit self'; }; }; dockerImageUbuntuWithTools = diff --git a/nix/systemModules/tests/test_envoy.py b/nix/systemModules/tests/test_envoy.py new file mode 100644 index 000000000..ee9759940 --- /dev/null +++ b/nix/systemModules/tests/test_envoy.py @@ -0,0 +1,3 @@ +def test_envoy_service(host): + assert host.service("envoy.service").is_valid + assert host.service("envoy.service").is_running From fea3e5361bd1b58a05fb221bb1ea83dab8758062 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 12 Aug 2025 20:11:40 +0200 Subject: [PATCH 2/3] Test envoy deployment with system manager We verify that the envoy service can be deployed using the system manager. We also verify that system manager deployment can be triggered from Ansible. --- ansible/tasks/setup-nix.yml | 11 ++++ ansible/tasks/setup-system-manager.yml | 7 +++ ansible/tests/conftest.py | 69 +++++++++++++++++++++++++- ansible/tests/nix.yaml | 5 ++ ansible/tests/test_nginx.py | 56 ++------------------- ansible/tests/test_nix.py | 13 +++++ flake.nix | 1 + nix/packages/ansible-test.nix | 55 +++----------------- nix/packages/default.nix | 5 +- nix/packages/envoy-bin.nix | 2 +- nix/systemConfigs.nix | 31 ++++++++++++ nix/systemModules/envoy.nix | 5 +- nix/systemModules/tests/default.nix | 11 +--- 13 files changed, 155 insertions(+), 116 deletions(-) create mode 100644 ansible/tasks/setup-nix.yml create mode 100644 ansible/tasks/setup-system-manager.yml create mode 100644 ansible/tests/nix.yaml create mode 100644 ansible/tests/test_nix.py create mode 100644 nix/systemConfigs.nix diff --git a/ansible/tasks/setup-nix.yml b/ansible/tasks/setup-nix.yml new file mode 100644 index 000000000..f31965bd6 --- /dev/null +++ b/ansible/tasks/setup-nix.yml @@ -0,0 +1,11 @@ +--- +- name: Check if nix is installed + ansible.builtin.command: which nix + register: nix_installed + failed_when: nix_installed.rc != 0 + ignore_errors: true + +- name: Install nix + ansible.builtin.shell: curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | sh -s -- install --no-confirm --extra-conf 'substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com' --extra-conf 'trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=% cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=' + when: nix_installed.rc != 0 + become: true diff --git a/ansible/tasks/setup-system-manager.yml b/ansible/tasks/setup-system-manager.yml new file mode 100644 index 000000000..720aad475 --- /dev/null +++ b/ansible/tasks/setup-system-manager.yml @@ -0,0 +1,7 @@ +--- +- name: Deploy system manager + ansible.builtin.shell: | + . /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh + cd /tmp + nix run /flake#system-manager -- switch --flake /flake + become: true diff --git a/ansible/tests/conftest.py b/ansible/tests/conftest.py index c993ff4cc..cfaba87fa 100644 --- a/ansible/tests/conftest.py +++ b/ansible/tests/conftest.py @@ -1,8 +1,16 @@ +import pytest +import subprocess +import testinfra +from rich.console import Console + +console = Console() + + def pytest_addoption(parser): parser.addoption( - "--ansible-dir", + "--flake-dir", action="store", - help="Directory containing Ansible playbooks and roles", + help="Directory containing the current flake", ) parser.addoption( @@ -10,3 +18,60 @@ def pytest_addoption(parser): action="store", help="Docker image and tag to use for testing", ) + + +@pytest.fixture(scope="module") +def host(request): + flake_dir = request.config.getoption("--flake-dir") + docker_id = ( + subprocess.check_output( + [ + "docker", + "run", + "--privileged", + "--cap-add", + "SYS_ADMIN", + "--security-opt", + "seccomp=unconfined", + "--cgroup-parent=docker.slice", + "--cgroupns", + "private", + "-v", + f"{flake_dir}:/flake", + "-d", + "ubuntu-cloudimg-with-tools:0.1", + ] + ) + .decode() + .strip() + ) + yield testinfra.get_host("docker://" + docker_id) + subprocess.check_call(["docker", "rm", "-f", docker_id], stdout=subprocess.DEVNULL) + + +@pytest.fixture(scope="module") +def run_ansible_playbook(host): + def _run_playbook(playbook_name, verbose=False): + cmd = [ + "ANSIBLE_HOST_KEY_CHECKING=False", + "ansible-playbook", + "--connection=local", + ] + if verbose: + cmd.append("-vvv") + cmd.extend([ + "-i", + "localhost,", + "--extra-vars", + "@/flake/ansible/vars.yml", + f"/flake/ansible/tests/{playbook_name}", + ]) + result = host.run(" ".join(cmd)) + if result.failed: + console.log(result.stdout) + console.log(result.stderr) + import pdb; pdb.set_trace() + raise pytest.fail( + f"Ansible playbook {playbook_name} failed with return code {result.rc}" + ) + return _run_playbook diff --git a/ansible/tests/nix.yaml b/ansible/tests/nix.yaml new file mode 100644 index 000000000..4effc67dd --- /dev/null +++ b/ansible/tests/nix.yaml @@ -0,0 +1,5 @@ +--- +- hosts: localhost + tasks: + - import_tasks: ../tasks/setup-nix.yml + - import_tasks: ../tasks/setup-system-manager.yml diff --git a/ansible/tests/test_nginx.py b/ansible/tests/test_nginx.py index cff5c0d69..ec68e82a9 100644 --- a/ansible/tests/test_nginx.py +++ b/ansible/tests/test_nginx.py @@ -1,59 +1,9 @@ import pytest -import subprocess -import testinfra -from rich.console import Console -console = Console() - -@pytest.fixture(scope="session") -def host(request): - ansible_dir = request.config.getoption("--ansible-dir") - docker_id = ( - subprocess.check_output( - [ - "docker", - "run", - "--privileged", - "--cap-add", - "SYS_ADMIN", - "--security-opt", - "seccomp=unconfined", - "--cgroup-parent=docker.slice", - "--cgroupns", - "private", - "-v", - f"{ansible_dir}/:/ansible/", - "-d", - "ubuntu-cloudimg-with-tools:0.1", - ] - ) - .decode() - .strip() - ) - yield testinfra.get_host("docker://" + docker_id) - subprocess.check_call(["docker", "rm", "-f", docker_id], stdout=subprocess.DEVNULL) - - -@pytest.fixture(scope="session", autouse=True) -def run_ansible(host): - cmd = [ - "ANSIBLE_HOST_KEY_CHECKING=False", - "ansible-playbook", - "--connection=local", - "-i", - "localhost,", - "--extra-vars", - "@/ansible/vars.yml", - "/ansible/tests/nginx.yaml", - ] - result = host.run(" ".join(cmd)) - if result.failed: - console.log(result.stdout) - console.log(result.stderr) - raise pytest.fail( - "Ansible playbook nginx.yaml failed with return code {}".format(result.rc) - ) +@pytest.fixture(scope="module", autouse=True) +def run_ansible(run_ansible_playbook): + run_ansible_playbook("nginx.yaml") def test_nginx_service(host): diff --git a/ansible/tests/test_nix.py b/ansible/tests/test_nix.py new file mode 100644 index 000000000..fe900664d --- /dev/null +++ b/ansible/tests/test_nix.py @@ -0,0 +1,13 @@ +import pytest + + +@pytest.fixture(scope="module", autouse=True) +def run_ansible(run_ansible_playbook): + run_ansible_playbook("nix.yaml", verbose=True) + + +def test_nix_service(host): + assert host.service("nix-daemon.service").is_running + +def test_envoy_service(host): + assert host.service("envoy.service").is_running diff --git a/flake.nix b/flake.nix index 78b2b383f..813ffc69f 100644 --- a/flake.nix +++ b/flake.nix @@ -59,6 +59,7 @@ nix/packages nix/overlays nix/systemModules + nix/systemConfigs.nix ]; }); } diff --git a/nix/packages/ansible-test.nix b/nix/packages/ansible-test.nix index c6d997b71..d82507408 100644 --- a/nix/packages/ansible-test.nix +++ b/nix/packages/ansible-test.nix @@ -1,48 +1,9 @@ { pkgs, lib, + docker-image-ubuntu, }: let - ubuntu-cloudimg = - let - cloudImg = builtins.fetchurl { - url = "http://cloud-images-archive.ubuntu.com/releases/noble/release-20250430/ubuntu-24.04-server-cloudimg-amd64-root.tar.xz"; - sha256 = "sha256:0rfi3qqs0sqarixfic7pzjpx7d4vldv2d98c5zjv7b90mirznvf9"; - }; - in - pkgs.runCommand "ubuntu-cloudimg" { nativeBuildInputs = [ pkgs.xz ]; } '' - mkdir -p $out - tar --exclude='dev/*' \ - --exclude='etc/systemd/system/network-online.target.wants/systemd-networkd-wait-online.service' \ - --exclude='etc/systemd/system/multi-user.target.wants/systemd-resolved.service' \ - --exclude='usr/lib/systemd/system/tpm-udev.service' \ - --exclude='usr/lib/systemd/system/systemd-remount-fs.service' \ - --exclude='usr/lib/systemd/system/systemd-resolved.service' \ - --exclude='var/lib/apt/lists/*' \ - -xJf ${cloudImg} -C $out - rm $out/bin $out/lib $out/lib64 $out/sbin - mkdir -p $out/run/systemd && echo 'docker' > $out/run/systemd/container - mkdir $out/var/lib/apt/lists/partial - ''; - - dockerImageUbuntu = pkgs.dockerTools.buildImage { - name = "ubuntu-cloudimg"; - tag = "0.1"; - created = "now"; - extraCommands = '' - ln -s usr/bin - ln -s usr/lib - ln -s usr/lib64 - ln -s usr/sbin - ''; - copyToRoot = pkgs.buildEnv { - name = "image-root"; - pathsToLink = [ "/" ]; - paths = [ ubuntu-cloudimg ]; - }; - config.Cmd = [ "/lib/systemd/systemd" ]; - }; - dockerImageUbuntuWithTools = let tools = [ pkgs.ansible ]; @@ -52,7 +13,8 @@ let tag = "0.1"; created = "now"; maxLayers = 30; - fromImage = dockerImageUbuntu; + fromImage = docker-image-ubuntu; + compressor = "zstd"; config = { Env = [ "PATH=${lib.makeBinPath tools}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" @@ -69,6 +31,7 @@ pkgs.writeShellApplication { requests pytest pytest-testinfra + pytest-xdist rich ] )) @@ -76,12 +39,10 @@ pkgs.writeShellApplication { text = '' echo "Running Ansible tests..." export DOCKER_IMAGE=${dockerImageUbuntuWithTools.imageName}:${dockerImageUbuntuWithTools.imageTag} - if ! docker image inspect $DOCKER_IMAGE > /dev/null; then - echo "Loading Docker image..." - docker load < ${dockerImageUbuntuWithTools} - fi - ANSIBLE_DIR=${../../ansible} - pytest -p no:cacheprovider -s -v "$@" $ANSIBLE_DIR/tests --ansible-dir=$ANSIBLE_DIR --docker-image=$DOCKER_IMAGE + echo "Loading Docker image..." + docker load < ${dockerImageUbuntuWithTools} + FLAKE_DIR=${../..} + pytest -x -p no:cacheprovider -s -v "$@" $FLAKE_DIR/ansible/tests --flake-dir=$FLAKE_DIR --docker-image=$DOCKER_IMAGE ''; meta = with pkgs.lib; { description = "Ansible test runner"; diff --git a/nix/packages/default.nix b/nix/packages/default.nix index ec8138086..45fbd775b 100644 --- a/nix/packages/default.nix +++ b/nix/packages/default.nix @@ -31,7 +31,9 @@ packages = ( { build-test-ami = pkgs.callPackage ./build-test-ami.nix { }; - ansible-test = pkgs.callPackage ./ansible-test.nix { }; + ansible-test = pkgs.callPackage ./ansible-test.nix { + inherit (self'.packages) docker-image-ubuntu; + }; cleanup-ami = pkgs.callPackage ./cleanup-ami.nix { }; dbmate-tool = pkgs.callPackage ./dbmate-tool.nix { inherit (self.supabase) defaults; }; docker-image-ubuntu = pkgs.callPackage ./docker-ubuntu.nix { }; @@ -61,6 +63,7 @@ name = "start-postgres-server"; }; sync-exts-versions = pkgs.callPackage ./sync-exts-versions.nix { inherit (inputs') nix-editor; }; + system-manager = inputs'.system-manager.packages.default; trigger-nix-build = pkgs.callPackage ./trigger-nix-build.nix { }; update-readme = pkgs.callPackage ./update-readme.nix { }; inherit (pkgs.callPackage ./wal-g.nix { }) wal-g-2 wal-g-3; diff --git a/nix/packages/envoy-bin.nix b/nix/packages/envoy-bin.nix index 5e47787b8..b26494a79 100644 --- a/nix/packages/envoy-bin.nix +++ b/nix/packages/envoy-bin.nix @@ -16,7 +16,7 @@ let .${system} or throwSystem; hash = { - aarch64-linux = ""; + aarch64-linux = "sha256-65MOMqtVVWQ+CdEdSQ45LQp5DFqA6wsOussQRr27EU0="; x86_64-linux = "sha256-JjlWPOm8CbHua9RzF2C1lsjtHkdM3YPMnfk2RRbhQ2c="; } .${system} or throwSystem; diff --git a/nix/systemConfigs.nix b/nix/systemConfigs.nix new file mode 100644 index 000000000..b66bff833 --- /dev/null +++ b/nix/systemConfigs.nix @@ -0,0 +1,31 @@ +{ self, inputs, ... }: +let + mkModules = system: [ + self.systemModules.envoy + ({ + services.nginx.enable = true; + nixpkgs.hostPlatform = system; + }) + ]; + + systems = [ + "aarch64-linux" + "x86_64-linux" + ]; + + mkSystemConfig = system: { + name = system; + value.default = inputs.system-manager.lib.makeSystemConfig { + modules = mkModules system; + extraSpecialArgs = { + inherit self; + inherit system; + }; + }; + }; +in +{ + flake = { + systemConfigs = builtins.listToAttrs (map mkSystemConfig systems); + }; +} diff --git a/nix/systemModules/envoy.nix b/nix/systemModules/envoy.nix index 457b2640e..4451f79ff 100644 --- a/nix/systemModules/envoy.nix +++ b/nix/systemModules/envoy.nix @@ -1,7 +1,8 @@ { lib, nixosModulesPath, - self', + self, + system, ... }: { @@ -11,7 +12,7 @@ config = { services.envoy = { enable = true; - package = self'.packages.envoy-bin; + package = self.packages.${system}.envoy-bin; # TODO: settings from postgres/ansible/files/envoy_config/ }; systemd.services.envoy = { diff --git a/nix/systemModules/tests/default.nix b/nix/systemModules/tests/default.nix index 99ced0c6d..104171a57 100644 --- a/nix/systemModules/tests/default.nix +++ b/nix/systemModules/tests/default.nix @@ -12,16 +12,7 @@ check-system-manager = let lib = pkgs.lib; - systemManagerConfig = self.inputs.system-manager.lib.makeSystemConfig { - modules = [ - self.systemModules.envoy - ({ - services.nginx.enable = true; - nixpkgs.hostPlatform = pkgs.system; - }) - ]; - extraSpecialArgs = { inherit self'; }; - }; + systemManagerConfig = self.systemConfigs.${pkgs.system}.default; dockerImageUbuntuWithTools = let From d2b986495d4ee8e63556e87e49c0841c2b0ba0d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 12 Aug 2025 20:31:51 +0200 Subject: [PATCH 3/3] ci: make sure we cache system manager and its test dependencies --- nix/checks.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nix/checks.nix b/nix/checks.nix index 2dc51797a..f311fedd8 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -314,6 +314,9 @@ inherit pkgs; }; devShell = self'.devShells.default; + } + // lib.optionalAttrs (pkgs.stdenv.hostPlatform.isLinux) { + inherit (self'.packages) ansible-test run-testinfra docker-image-ubuntu; }; }; }