Skip to content

Commit 08f30d1

Browse files
mcgrofmcgrof
committed
mirror: add Nix binary cache mirroring support
Add support for a Nix binary cache mirroring infrastructure for enterprise environments to reduce latencies and public bandwidth consumption for CIs. This dramatically reduces NixOS package download times from hundreds of packages to local cache hits. Key features: - nginx-based proxy caching for Nix binary cache - Automatic mirror detection for client-side configuration - systemd service and timer for cache synchronization - Integration with existing defconfig-mirror infrastructure - Client auto-configuration when local mirrors are available Components: - playbooks/roles/nix-cache-mirror/: Complete Ansible role - scripts/check_nix_mirror.sh: Mirror detection and URL discovery - kconfigs/Kconfig.mirror: Mirror configuration options - Integration with defconfig-mirror for one-time setup The mirror setup follows existing kdevops mirror patterns and integrates with the git mirror infrastructure already in place. Generated-by: Claude AI Signed-off-by: Luis Chamberlain <[email protected]>
1 parent 2187526 commit 08f30d1

File tree

11 files changed

+424
-2
lines changed

11 files changed

+424
-2
lines changed

‎defconfigs/mirror‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,4 @@ CONFIG_SKIP_BRINGUP=y
22
CONFIG_WORKFLOWS=n
33
CONFIG_INSTALL_LOCAL_LINUX_MIRROR=y
44
CONFIG_LINUX_MIRROR_NFS=y
5+
CONFIG_INSTALL_NIX_CACHE_MIRROR=y

‎kconfigs/Kconfig.mirror‎

Lines changed: 66 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -379,15 +379,15 @@ choice
379379
config MIRROR_STABLE_RC_HTTPS
380380
bool "HTTPS (kernel.org)"
381381
help
382-
If you enable this option then the mirror will use HTTPS to access
382+
If you enable this option then the mirror will use HTTPS to access
383383
the linux-stable-rc repository on git.kernel.org. The full URL is:
384384

385385
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable-rc.git"
386386

387387
config MIRROR_STABLE_RC_HTTPS_GOOGLE
388388
bool "HTTPS (Google)"
389389
help
390-
If you enable this option then the mirror will use HTTPS to access
390+
If you enable this option then the mirror will use HTTPS to access
391391
the linux-stable-rc repository on kernel.googlesource.com The full
392392
URL is:
393393

@@ -676,5 +676,69 @@ config MIRROR_MMTESTS_URL
676676
string
677677
default DEFAULT_MMTESTS_GITHUB_HTTPS_URL if MIRROR_MMTESTS_HTTPS_GITHUB
678678

679+
config INSTALL_NIX_CACHE_MIRROR
680+
bool "Install Nix binary cache mirror"
681+
output yaml
682+
depends on INSTALL_LOCAL_LINUX_MIRROR
683+
help
684+
Enable this to set up a local Nix binary cache mirror for all guests.
685+
This will significantly speed up NixOS VM builds by caching downloaded
686+
packages locally.
687+
688+
When enabled, this creates:
689+
- A local Nix binary cache at /mirror/nix-cache/
690+
- An nginx-based cache server on port 8080
691+
- Automatic synchronization from cache.nixos.org
692+
693+
This saves bandwidth and speeds up NixOS builds dramatically as the
694+
initial build requires downloading ~679 packages (685MB).
695+
696+
The cache will be available to all VMs and systems in your network
697+
at: http://mirror-host:8080/
698+
699+
choice
700+
prompt "Nix cache mirror source"
701+
default MIRROR_NIX_CACHE_NIXOS_ORG
702+
depends on INSTALL_NIX_CACHE_MIRROR
703+
704+
config MIRROR_NIX_CACHE_NIXOS_ORG
705+
bool "cache.nixos.org (official)"
706+
help
707+
Use the official Nix binary cache at cache.nixos.org as the upstream
708+
source for the mirror.
709+
710+
config MIRROR_NIX_CACHE_NIXOS_ORG_FALLBACK
711+
bool "cache.nixos.org with fastly CDN fallback"
712+
help
713+
Use cache.nixos.org with additional fastly CDN endpoints as fallbacks
714+
for better reliability and performance.
715+
716+
endchoice
717+
718+
config MIRROR_NIX_CACHE_UPSTREAM_URL
719+
string
720+
output yaml
721+
default "https://cache.nixos.org" if MIRROR_NIX_CACHE_NIXOS_ORG
722+
default "https://cache.nixos.org https://nixos.cachix.org" if MIRROR_NIX_CACHE_NIXOS_ORG_FALLBACK
723+
724+
config NIX_CACHE_MIRROR_PORT
725+
int "Nix cache mirror nginx port"
726+
output yaml
727+
default 8080
728+
depends on INSTALL_NIX_CACHE_MIRROR
729+
help
730+
The port for the local nginx server that will serve the Nix binary cache.
731+
Default is 8080. Ensure this port is available and not blocked by firewall.
732+
733+
config NIX_CACHE_MIRROR_PATH
734+
string "Nix cache mirror storage path"
735+
output yaml
736+
default "/mirror/nix-cache"
737+
depends on INSTALL_NIX_CACHE_MIRROR
738+
help
739+
Local filesystem path where the Nix binary cache will be stored.
740+
This should be on a filesystem with sufficient space as the cache
741+
can grow to several GB over time.
742+
679743
endif # ENABLE_LOCAL_LINUX_MIRROR
680744
endif # TERRAFORM

‎kconfigs/Kconfig.nixos‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,31 @@ config NIXOS_SSH_PORT
9191
help
9292
SSH port to use for connecting to NixOS VMs.
9393

94+
config NIXOS_USE_LOCAL_MIRROR
95+
bool "Use local Nix binary cache mirror"
96+
output yaml
97+
default $(shell, scripts/check_nix_mirror.sh USE_NIX_CACHE_MIRROR)
98+
help
99+
Enable this to use a local Nix binary cache mirror if available.
100+
This dramatically speeds up NixOS builds by avoiding repeated
101+
downloads of packages from cache.nixos.org.
102+
103+
When enabled, kdevops will automatically detect and configure
104+
VMs to use the local mirror if available at /mirror/nix-cache
105+
or via HTTP on port 8080.
106+
107+
This requires that you have set up a mirror server using the
108+
defconfig-mirror configuration.
109+
110+
config NIXOS_MIRROR_URL
111+
string "Nix binary cache mirror URL"
112+
output yaml
113+
default ""
114+
depends on NIXOS_USE_LOCAL_MIRROR
115+
help
116+
The URL of the local Nix binary cache mirror. This will be
117+
automatically detected at runtime when available.
118+
94119
config NIXOS_DEBUG_MODE
95120
bool "Enable debug mode for NixOS provisioning"
96121
default n

‎playbooks/linux-mirror.yml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,18 @@
11
---
22
- name: Install Git mirrors using systemd.timer
33
hosts: localhost
4+
pre_tasks:
5+
- name: Check if Nix mirror directory exists
6+
ansible.builtin.stat:
7+
path: /mirror/nix-cache
8+
register: nix_mirror_dir_check
9+
when: install_nix_cache_mirror | default(false) | bool
10+
11+
- name: Set nix mirror available fact
12+
ansible.builtin.set_fact:
13+
nix_mirror_available: "{{ install_nix_cache_mirror | default(false) | bool and nix_mirror_dir_check.stat.exists | default(false) }}"
14+
415
roles:
516
- role: linux-mirror
17+
- role: nix-cache-mirror
18+
when: nix_mirror_available | default(false) | bool

‎playbooks/roles/nix-cache-mirror/defaults/main.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
---
2+
# Default variables for nix-cache-mirror role
3+
nix_cache_mirror_path: "/mirror/nix-cache"
4+
nix_cache_mirror_port: 8080
5+
nix_cache_upstream_url: "https://cache.nixos.org"
6+
nix_cache_mirror_nginx_conf_path: "/etc/nginx/sites-available/nix-cache-mirror"
7+
nix_cache_mirror_nginx_enabled_path: "/etc/nginx/sites-enabled/nix-cache-mirror"

‎playbooks/roles/nix-cache-mirror/handlers/main.yml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
---
2+
- name: Reload nginx
3+
become: true
4+
ansible.builtin.systemd:
5+
name: nginx
6+
state: reloaded
7+
8+
- name: Reload systemd
9+
become: true
10+
ansible.builtin.systemd:
11+
daemon_reload: true

‎playbooks/roles/nix-cache-mirror/tasks/main.yml‎

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
---
2+
- name: Import optional extra_args file
3+
ansible.builtin.include_vars: "{{ item }}"
4+
ignore_errors: true
5+
with_first_found:
6+
- files:
7+
- "../extra_vars.yml"
8+
- "../extra_vars.yaml"
9+
- "../extra_vars.json"
10+
skip: true
11+
tags: vars
12+
13+
- name: Fail if nix cache mirror is enabled but user is not root
14+
ansible.builtin.fail:
15+
msg: "Nix cache mirror setup requires root privileges. Please run as root."
16+
when:
17+
- install_nix_cache_mirror | bool
18+
- ansible_user_id != 'root'
19+
tags: ["nix-cache", "mirror"]
20+
21+
- name: Install nginx for Nix cache mirror
22+
become: true
23+
ansible.builtin.package:
24+
name:
25+
- nginx
26+
- curl
27+
state: present
28+
when: install_nix_cache_mirror | bool
29+
tags: ["nix-cache", "mirror"]
30+
31+
- name: Create Nix cache mirror directory
32+
become: true
33+
ansible.builtin.file:
34+
path: "{{ nix_cache_mirror_path }}"
35+
state: directory
36+
mode: "0755"
37+
owner: www-data
38+
group: www-data
39+
when: install_nix_cache_mirror | bool
40+
tags: ["nix-cache", "mirror"]
41+
42+
- name: Template nginx configuration for Nix cache mirror
43+
become: true
44+
ansible.builtin.template:
45+
src: nix-cache-mirror.nginx.j2
46+
dest: "{{ nix_cache_mirror_nginx_conf_path }}"
47+
mode: "0644"
48+
owner: root
49+
group: root
50+
when: install_nix_cache_mirror | bool
51+
notify: reload nginx
52+
tags: ["nix-cache", "mirror"]
53+
54+
- name: Enable nginx site for Nix cache mirror
55+
become: true
56+
ansible.builtin.file:
57+
src: "{{ nix_cache_mirror_nginx_conf_path }}"
58+
dest: "{{ nix_cache_mirror_nginx_enabled_path }}"
59+
state: link
60+
when: install_nix_cache_mirror | bool
61+
notify: reload nginx
62+
tags: ["nix-cache", "mirror"]
63+
64+
- name: Remove default nginx site
65+
become: true
66+
ansible.builtin.file:
67+
path: /etc/nginx/sites-enabled/default
68+
state: absent
69+
when: install_nix_cache_mirror | bool
70+
notify: reload nginx
71+
tags: ["nix-cache", "mirror"]
72+
73+
- name: Create systemd service for Nix cache syncer
74+
become: true
75+
ansible.builtin.template:
76+
src: nix-cache-sync.service.j2
77+
dest: /etc/systemd/system/nix-cache-sync.service
78+
mode: "0644"
79+
when: install_nix_cache_mirror | bool
80+
notify: reload systemd
81+
tags: ["nix-cache", "mirror"]
82+
83+
- name: Create systemd timer for Nix cache syncer
84+
become: true
85+
ansible.builtin.template:
86+
src: nix-cache-sync.timer.j2
87+
dest: /etc/systemd/system/nix-cache-sync.timer
88+
mode: "0644"
89+
when: install_nix_cache_mirror | bool
90+
notify: reload systemd
91+
tags: ["nix-cache", "mirror"]
92+
93+
- name: Enable and start nginx service
94+
become: true
95+
ansible.builtin.systemd:
96+
name: nginx
97+
enabled: true
98+
state: started
99+
daemon_reload: true
100+
when: install_nix_cache_mirror | bool
101+
tags: ["nix-cache", "mirror"]
102+
103+
- name: Enable and start nix-cache-sync timer
104+
become: true
105+
ansible.builtin.systemd:
106+
name: nix-cache-sync.timer
107+
enabled: true
108+
state: started
109+
daemon_reload: true
110+
when: install_nix_cache_mirror | bool
111+
tags: ["nix-cache", "mirror"]
112+
113+
- name: Check if firewalld is running
114+
ansible.builtin.command: systemctl is-active firewalld
115+
register: firewalld_status
116+
ignore_errors: true
117+
when:
118+
- install_nix_cache_mirror | bool
119+
- linux_mirror_nfs | bool
120+
tags: ["nix-cache", "mirror"]
121+
122+
- name: Open firewall for Nix cache mirror HTTP traffic
123+
become: true
124+
ansible.posix.firewalld:
125+
port: "{{ nix_cache_mirror_port }}/tcp"
126+
permanent: true
127+
state: enabled
128+
immediate: true
129+
when:
130+
- install_nix_cache_mirror | bool
131+
- linux_mirror_nfs | bool
132+
- firewalld_status.rc == 0
133+
tags: ["nix-cache", "mirror"]

‎playbooks/roles/nix-cache-mirror/templates/nix-cache-mirror.nginx.j2‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
server {
2+
listen {{ nix_cache_mirror_port }};
3+
server_name _;
4+
5+
# Cache location
6+
root {{ nix_cache_mirror_path }};
7+
8+
# Enable directory listings
9+
autoindex on;
10+
autoindex_exact_size off;
11+
autoindex_localtime on;
12+
13+
# Security headers
14+
add_header X-Frame-Options "SAMEORIGIN" always;
15+
add_header X-Content-Type-Options "nosniff" always;
16+
add_header X-XSS-Protection "1; mode=block" always;
17+
18+
# Logging
19+
access_log /var/log/nginx/nix-cache-mirror.access.log;
20+
error_log /var/log/nginx/nix-cache-mirror.error.log;
21+
22+
# Main location for cached files
23+
location / {
24+
try_files $uri $uri/ @upstream;
25+
26+
# Cache headers for clients
27+
expires 1d;
28+
add_header Cache-Control "public, immutable";
29+
30+
# Enable range requests for large files
31+
location ~* \.(nar|narinfo)$ {
32+
add_header Accept-Ranges bytes;
33+
}
34+
}
35+
36+
# Upstream fallback for cache misses
37+
location @upstream {
38+
# Proxy to upstream cache with caching
39+
proxy_pass {{ mirror_nix_cache_upstream_url | default('https://cache.nixos.org') }};
40+
proxy_set_header Host cache.nixos.org;
41+
proxy_set_header X-Real-IP $remote_addr;
42+
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
43+
proxy_set_header X-Forwarded-Proto $scheme;
44+
45+
# Cache configuration
46+
proxy_cache_path {{ nix_cache_mirror_path }}/nginx-cache levels=1:2 keys_zone=nixcache:100m max_size=50g inactive=7d use_temp_path=off;
47+
proxy_cache nixcache;
48+
proxy_cache_valid 200 206 301 302 7d;
49+
proxy_cache_valid 404 1m;
50+
proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504;
51+
proxy_cache_lock on;
52+
proxy_cache_lock_timeout 5s;
53+
54+
# Store successful requests to filesystem
55+
proxy_store on;
56+
proxy_store_access user:rw group:rw all:r;
57+
proxy_temp_path {{ nix_cache_mirror_path }}/tmp;
58+
59+
# Add headers to indicate cache status
60+
add_header X-Cache-Status $upstream_cache_status;
61+
add_header X-Cache-Date $upstream_http_date;
62+
}
63+
64+
# Status endpoint
65+
location /status {
66+
access_log off;
67+
return 200 "Nix Cache Mirror OK\n";
68+
add_header Content-Type text/plain;
69+
}
70+
71+
# Health check
72+
location /health {
73+
access_log off;
74+
return 200 "healthy\n";
75+
add_header Content-Type text/plain;
76+
}
77+
}

‎playbooks/roles/nix-cache-mirror/templates/nix-cache-sync.service.j2‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
[Unit]
2+
Description=Nix Binary Cache Sync Service
3+
After=network-online.target
4+
Wants=network-online.target
5+
6+
[Service]
7+
Type=oneshot
8+
User=www-data
9+
Group=www-data
10+
ExecStart=/usr/bin/curl -f -s "{{ mirror_nix_cache_upstream_url | default('https://cache.nixos.org') }}/nix-cache-info" -o "{{ nix_cache_mirror_path }}/nix-cache-info"
11+
ExecStartPost=/bin/bash -c 'find {{ nix_cache_mirror_path }} -name "*.narinfo" -mtime +7 -delete'
12+
ExecStartPost=/bin/bash -c 'find {{ nix_cache_mirror_path }} -name "*.nar*" -mtime +7 -delete'
13+
StandardOutput=journal
14+
StandardError=journal
15+
16+
[Install]
17+
WantedBy=multi-user.target

0 commit comments

Comments
 (0)