Skip to content

Commit 3b68685

Browse files
mcgrofmcgrof
committed
mirror: add Nix binary cache mirroring support
Add support for a Nix binary cache mirroring infrastructure for enterprise environments to reduce latencies and public bandwidth consumption for CIs. This dramatically reduces NixOS package download times from hundreds of packages to local cache hits. Key features: - nginx-based proxy caching for Nix binary cache - Automatic mirror detection for client-side configuration - systemd service and timer for cache synchronization - Integration with existing defconfig-mirror infrastructure - Client auto-configuration when local mirrors are available Components: - playbooks/roles/nix-cache-mirror/: Complete Ansible role - scripts/check_nix_mirror.sh: Mirror detection and URL discovery - kconfigs/Kconfig.mirror: Mirror configuration options - Integration with defconfig-mirror for one-time setup The mirror setup follows existing kdevops mirror patterns and integrates with the git mirror infrastructure already in place. Generated-by: Claude AI Signed-off-by: Luis Chamberlain <[email protected]>
1 parent 3089c3f commit 3b68685

File tree

14 files changed

+507
-7
lines changed

14 files changed

+507
-7
lines changed

‎defconfigs/mirror‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,4 @@ CONFIG_SKIP_BRINGUP=y
22
CONFIG_WORKFLOWS=n
33
CONFIG_INSTALL_LOCAL_LINUX_MIRROR=y
44
CONFIG_LINUX_MIRROR_NFS=y
5+
CONFIG_INSTALL_NIX_CACHE_MIRROR=y

‎kconfigs/Kconfig.mirror‎

Lines changed: 66 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -379,15 +379,15 @@ choice
379379
config MIRROR_STABLE_RC_HTTPS
380380
bool "HTTPS (kernel.org)"
381381
help
382-
If you enable this option then the mirror will use HTTPS to access
382+
If you enable this option then the mirror will use HTTPS to access
383383
the linux-stable-rc repository on git.kernel.org. The full URL is:
384384

385385
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable-rc.git"
386386

387387
config MIRROR_STABLE_RC_HTTPS_GOOGLE
388388
bool "HTTPS (Google)"
389389
help
390-
If you enable this option then the mirror will use HTTPS to access
390+
If you enable this option then the mirror will use HTTPS to access
391391
the linux-stable-rc repository on kernel.googlesource.com The full
392392
URL is:
393393

@@ -676,5 +676,69 @@ config MIRROR_MMTESTS_URL
676676
string
677677
default DEFAULT_MMTESTS_GITHUB_HTTPS_URL if MIRROR_MMTESTS_HTTPS_GITHUB
678678

679+
config INSTALL_NIX_CACHE_MIRROR
680+
bool "Install Nix binary cache mirror"
681+
output yaml
682+
depends on INSTALL_LOCAL_LINUX_MIRROR
683+
help
684+
Enable this to set up a local Nix binary cache mirror for all guests.
685+
This will significantly speed up NixOS VM builds by caching downloaded
686+
packages locally.
687+
688+
When enabled, this creates:
689+
- A local Nix binary cache at /mirror/nix-cache/
690+
- An nginx-based cache server on port 8080
691+
- Automatic synchronization from cache.nixos.org
692+
693+
This saves bandwidth and speeds up NixOS builds dramatically as the
694+
initial build requires downloading ~679 packages (685MB).
695+
696+
The cache will be available to all VMs and systems in your network
697+
at: http://mirror-host:8080/
698+
699+
choice
700+
prompt "Nix cache mirror source"
701+
default MIRROR_NIX_CACHE_NIXOS_ORG
702+
depends on INSTALL_NIX_CACHE_MIRROR
703+
704+
config MIRROR_NIX_CACHE_NIXOS_ORG
705+
bool "cache.nixos.org (official)"
706+
help
707+
Use the official Nix binary cache at cache.nixos.org as the upstream
708+
source for the mirror.
709+
710+
config MIRROR_NIX_CACHE_NIXOS_ORG_FALLBACK
711+
bool "cache.nixos.org with fastly CDN fallback"
712+
help
713+
Use cache.nixos.org with additional fastly CDN endpoints as fallbacks
714+
for better reliability and performance.
715+
716+
endchoice
717+
718+
config MIRROR_NIX_CACHE_UPSTREAM_URL
719+
string
720+
output yaml
721+
default "https://cache.nixos.org" if MIRROR_NIX_CACHE_NIXOS_ORG
722+
default "https://cache.nixos.org https://nixos.cachix.org" if MIRROR_NIX_CACHE_NIXOS_ORG_FALLBACK
723+
724+
config NIX_CACHE_MIRROR_PORT
725+
int "Nix cache mirror nginx port"
726+
output yaml
727+
default 8080
728+
depends on INSTALL_NIX_CACHE_MIRROR
729+
help
730+
The port for the local nginx server that will serve the Nix binary cache.
731+
Default is 8080. Ensure this port is available and not blocked by firewall.
732+
733+
config NIX_CACHE_MIRROR_PATH
734+
string "Nix cache mirror storage path"
735+
output yaml
736+
default "/mirror/nix-cache"
737+
depends on INSTALL_NIX_CACHE_MIRROR
738+
help
739+
Local filesystem path where the Nix binary cache will be stored.
740+
This should be on a filesystem with sufficient space as the cache
741+
can grow to several GB over time.
742+
679743
endif # ENABLE_LOCAL_LINUX_MIRROR
680744
endif # TERRAFORM

‎kconfigs/Kconfig.nixos‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,35 @@ config NIXOS_SSH_PORT
9191
help
9292
SSH port to use for connecting to NixOS VMs.
9393

94+
config NIXOS_USE_LOCAL_MIRROR
95+
bool "Use local Nix binary cache mirror"
96+
output yaml
97+
default $(shell, scripts/check_nix_mirror.sh USE_NIX_CACHE_MIRROR)
98+
help
99+
Enable this to use a local Nix binary cache mirror if available.
100+
This dramatically speeds up NixOS builds by avoiding repeated
101+
downloads of packages from cache.nixos.org.
102+
103+
When enabled, kdevops will automatically detect and configure
104+
VMs to use the local mirror if available at /mirror/nix-cache
105+
or via HTTP on port 8080.
106+
107+
This requires that you have set up a mirror server using the
108+
defconfig-mirror configuration.
109+
110+
config NIXOS_MIRROR_URL
111+
string "Nix binary cache mirror URL"
112+
output yaml
113+
default $(shell, scripts/check_nix_mirror.sh NIX_CACHE_MIRROR_URL)
114+
depends on NIXOS_USE_LOCAL_MIRROR
115+
help
116+
The URL of the local Nix binary cache mirror. This is
117+
automatically detected based on available mirrors:
118+
- HTTP mirror if running on port 8080
119+
- File-based mirror at /mirror/nix-cache (for NFS mounts)
120+
121+
Leave empty to auto-detect at runtime.
122+
94123
config NIXOS_DEBUG_MODE
95124
bool "Enable debug mode for NixOS provisioning"
96125
default n

‎playbooks/linux-mirror.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,5 @@
33
hosts: localhost
44
roles:
55
- role: linux-mirror
6+
- role: nix-cache-mirror
7+
when: install_nix_cache_mirror | default(false) | bool

‎playbooks/nixos.yml‎

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,21 @@
9595
ansible.builtin.set_fact:
9696
nixos_ssh_authorized_key: "{{ ssh_public_key['content'] | b64decode | trim }}"
9797

98+
- name: Detect local Nix cache mirror URL if enabled
99+
ansible.builtin.shell: |
100+
bash {{ playbook_dir }}/../scripts/check_nix_mirror.sh NIX_CACHE_MIRROR_URL
101+
register: detected_mirror_url
102+
when: nixos_use_local_mirror | default(false) | bool and (nixos_mirror_url is not defined or nixos_mirror_url == "")
103+
changed_when: false
104+
105+
- name: Set detected mirror URL
106+
ansible.builtin.set_fact:
107+
nixos_mirror_url: "{{ detected_mirror_url.stdout | trim }}"
108+
when:
109+
- detected_mirror_url is defined
110+
- detected_mirror_url.stdout is defined
111+
- detected_mirror_url.stdout | trim != ""
112+
98113
- name: Template base NixOS configuration
99114
ansible.builtin.template:
100115
src: nixos/configuration.nix.j2
@@ -176,9 +191,18 @@
176191
fi
177192
178193
# Configure Nix to use local mirror if available
179-
{% if nixos_use_local_mirror is defined and nixos_use_local_mirror and nixos_mirror_url is defined and nixos_mirror_url != "" %}
194+
{% if nixos_use_local_mirror is defined and nixos_use_local_mirror %}
195+
# Dynamically detect mirror URL if not provided
196+
{% if nixos_mirror_url is not defined or nixos_mirror_url == "" %}
197+
DETECTED_MIRROR=$(bash {{ playbook_dir }}/../scripts/check_nix_mirror.sh NIX_CACHE_MIRROR_URL)
198+
if [ -n "$DETECTED_MIRROR" ]; then
199+
export NIX_CONFIG="substituters = $DETECTED_MIRROR https://cache.nixos.org"
200+
echo "Using detected local Nix cache mirror: $DETECTED_MIRROR"
201+
fi
202+
{% else %}
180203
export NIX_CONFIG="substituters = {{ nixos_mirror_url }} https://cache.nixos.org"
181-
echo "Using local Nix cache mirror: {{ nixos_mirror_url }}"
204+
echo "Using configured local Nix cache mirror: {{ nixos_mirror_url }}"
205+
{% endif %}
182206
{% endif %}
183207
184208
cd {{ nixos_generation_dir }}

‎playbooks/roles/linux-mirror/python/start-mirroring.py‎

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,8 @@ def mirror_entry(mirror, args):
5454
cmd = cmd + reference_args
5555
mirror_target = mirror_path + target
5656
if os.path.isdir(mirror_target):
57-
return
57+
sys.stdout.write("Skipping %s - mirror already exists at %s\n" % (short_name, mirror_target))
58+
return "skipped"
5859
sys.stdout.write("Mirroring: %s onto %s\n" % (short_name, mirror_target))
5960
if args.verbose:
6061
sys.stdout.write("%s\n" % (cmd))
@@ -74,6 +75,7 @@ def mirror_entry(mirror, args):
7475
process.wait()
7576
if process.returncode != 0:
7677
raise Exception(f"Failed clone with:\n%s" % (" ".join(cmd)))
78+
return "cloned"
7779

7880

7981
def main():
@@ -134,15 +136,36 @@ def main():
134136
% (mirror.get("short_name"), args.yaml_mirror, total)
135137
)
136138

139+
# Statistics tracking
140+
stats = {"skipped": 0, "cloned": 0, "failed": 0}
137141
# Mirror trees without a reference first
138142
for mirror in yaml_vars["mirrors"]:
139143
if not mirror.get("reference"):
140-
mirror_entry(mirror, args)
144+
result = mirror_entry(mirror, args)
145+
if result == "skipped":
146+
stats["skipped"] += 1
147+
elif result == "cloned":
148+
stats["cloned"] += 1
149+
else:
150+
stats["failed"] += 1
141151

142152
# Mirror trees which need a reference last
143153
for mirror in yaml_vars["mirrors"]:
144154
if mirror.get("reference"):
145-
mirror_entry(mirror, args)
155+
result = mirror_entry(mirror, args)
156+
if result == "skipped":
157+
stats["skipped"] += 1
158+
elif result == "cloned":
159+
stats["cloned"] += 1
160+
else:
161+
stats["failed"] += 1
162+
# Print summary
163+
sys.stdout.write("\n=== Mirror Summary ===\n")
164+
sys.stdout.write("Total repositories: %d\n" % total)
165+
sys.stdout.write("Skipped (already exist): %d\n" % stats["skipped"])
166+
sys.stdout.write("Cloned (new): %d\n" % stats["cloned"])
167+
if stats["failed"] > 0:
168+
sys.stdout.write("Failed: %d\n" % stats["failed"])
146169

147170

148171
if __name__ == "__main__":

‎playbooks/roles/nix-cache-mirror/defaults/main.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
---
2+
# Default variables for nix-cache-mirror role
3+
nix_cache_mirror_path: "/mirror/nix-cache"
4+
nix_cache_mirror_port: 8080
5+
nix_cache_upstream_url: "https://cache.nixos.org"
6+
nix_cache_mirror_nginx_conf_path: "/etc/nginx/sites-available/nix-cache-mirror"
7+
nix_cache_mirror_nginx_enabled_path: "/etc/nginx/sites-enabled/nix-cache-mirror"

‎playbooks/roles/nix-cache-mirror/handlers/main.yml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
---
2+
- name: reload nginx
3+
become: true
4+
ansible.builtin.systemd:
5+
name: nginx
6+
state: reloaded
7+
8+
- name: reload systemd
9+
become: true
10+
ansible.builtin.systemd:
11+
daemon_reload: true

0 commit comments

Comments
 (0)