Skip to content

Commit e3429e3

Browse files
cpu
1 parent 71c382c commit e3429e3

File tree

10 files changed

+1017
-297
lines changed

10 files changed

+1017
-297
lines changed

desktop/l/home.nix

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -465,9 +465,6 @@
465465
# Filesystem/Monitoring
466466
inotify-tools
467467

468-
# Printing
469-
hplip
470-
#hplipWithPlugin
471468

472469
# SDR
473470
gnuradio

desktop/l2/CPU_and_IRQ_optimization.md

Lines changed: 329 additions & 160 deletions
Large diffs are not rendered by default.

desktop/l2/configuration.nix

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@
4343
./irq-affinity.nix
4444
./systemd-slices.nix
4545
./kernel-params.nix
46-
./monitoring.nix
46+
#./monitoring.nix
4747
];
4848

4949
boot = {
@@ -181,6 +181,9 @@
181181

182182
system.stateVersion = "24.11";
183183

184+
systemd.extraConfig = "CPUAffinity=8,20,9,21,10,22,11,23";
185+
systemd.user.extraConfig = "CPUAffinity=8,20,9,21,10,22,11,23";
186+
184187
}
185188

186189
# end

desktop/l2/hostapd-multi.nix

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,6 @@ in {
127127
}
128128
];
129129
};
130-
serviceConfig.Slice = "kea.slice";
131130
};
132131
};
133132
# services.prometheus.exporters.kea = {
@@ -149,7 +148,6 @@ in {
149148
serve_rfc1918 = true;
150149
};
151150
};
152-
serviceConfig.Slice = "pdns.slice";
153151
};
154152

155153
# IPv6 SLAAC via radvd
@@ -168,7 +166,6 @@ in {
168166
};
169167
};
170168
'';
171-
serviceConfig.Slice = "radvd.slice";
172169
};
173170

174171
# https://nixos.wiki/wiki/Systemd-networkd

desktop/l2/irq-affinity.nix

Lines changed: 79 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -5,32 +5,96 @@
55

66
let
77
# Network core assignments (cache-aware, paired SMT siblings)
8-
networkCores = "0,12,1,13,2,14,3,15,4,16,5,17,6,18,7,19"; # Dedicated network processing cores
8+
# Ethernet IRQs: cores 0,12,1,13,2,14,3,15 (first 4 L cores)
9+
# WiFi IRQs: cores 4,5,6,7 (dedicated L cores for default queues)
10+
# Userland: cores 8,20,9,21,10,22,11,23 (remaining 4 L cores)
11+
ethernetCores = "0,12,1,13,2,14,3,15"; # Ethernet IRQ cores
12+
wifiCores = "4,5,6,7"; # WiFi default queue cores (L cores)
13+
userlandCores = "8,20,9,21,10,22,11,23"; # Userland cores
914

10-
# IRQ affinity script (distribute across paired SMT siblings)
15+
# IRQ affinity script (optimize WiFi default queues across dedicated L cores)
1116
irqAffinityScript = pkgs.writeShellScript "irq-affinity" ''
1217
#!/bin/bash
1318
set -euo pipefail
1419
15-
echo "Setting IRQ affinity for network optimization (cache-aware, paired SMT siblings)..."
20+
echo "Setting IRQ affinity for network optimization..."
1621
17-
# List of network cores (paired SMT siblings)
18-
network_cores=(0 12 1 13 2 14 3 15 4 16 5 17 6 18 7 19)
22+
# Ethernet cores (first 4 L cores)
23+
ethernet_cores="0 12 1 13 2 14 3 15"
24+
# WiFi default queue cores (L cores 4,5,6,7)
25+
wifi_default_cores="4 5 6 7"
26+
27+
# Distribute Ethernet IRQs across ethernet cores
28+
echo "Distributing Ethernet IRQs across cores: ${ethernetCores}"
1929
irq_index=0
20-
for irq in $(grep -E '(enp|wlp)' /proc/interrupts | awk '{print $1}' | sed 's/://'); do
21-
cpu=${network_cores[$((irq_index % ${#network_cores[@]}))]}
22-
if [[ -e "/proc/irq/$irq/smp_affinity_list" ]]; then
23-
echo "$cpu" > "/proc/irq/$irq/smp_affinity_list"
24-
echo "IRQ $irq -> CPU $cpu"
25-
else
26-
echo "Warning: IRQ $irq not found"
27-
fi
28-
irq_index=$((irq_index + 1))
30+
for interface in $(${pkgs.iproute2}/bin/ip link show | ${pkgs.gnugrep}/bin/grep -E "enp|eth" | ${pkgs.gawk}/bin/awk -F: '{print $2}' | ${pkgs.gnused}/bin/sed 's/ //g'); do
31+
for irq in $(${pkgs.gnugrep}/bin/grep "$interface" /proc/interrupts | ${pkgs.gawk}/bin/awk '{print $1}' | ${pkgs.gnused}/bin/sed 's/://'); do
32+
cpu_index=$((irq_index % 8))
33+
# Convert index to actual CPU number
34+
case $cpu_index in
35+
0) cpu=0 ;;
36+
1) cpu=12 ;;
37+
2) cpu=1 ;;
38+
3) cpu=13 ;;
39+
4) cpu=2 ;;
40+
5) cpu=14 ;;
41+
6) cpu=3 ;;
42+
7) cpu=15 ;;
43+
esac
44+
if [[ -e "/proc/irq/$irq/smp_affinity_list" ]]; then
45+
echo "$cpu" > "/proc/irq/$irq/smp_affinity_list"
46+
echo "Ethernet IRQ $irq ($interface) -> CPU $cpu"
47+
else
48+
echo "Warning: Ethernet IRQ $irq not found"
49+
fi
50+
irq_index=$((irq_index + 1))
51+
done
2952
done
3053
54+
# Optimize WiFi default queues across dedicated L cores
55+
echo "Optimizing WiFi default queues across cores: ${wifiCores}"
56+
57+
# Get all WiFi default queue IRQs by PCI device
58+
wifi_default_irqs=$(cat /proc/interrupts | ${pkgs.gnugrep}/bin/grep "iwlwifi:default_queue" | ${pkgs.gawk}/bin/awk '{print $1}' | ${pkgs.gnused}/bin/sed 's/://')
59+
60+
if [ -n "$wifi_default_irqs" ]; then
61+
wifi_count=0
62+
for irq in $wifi_default_irqs; do
63+
# Assign each WiFi default queue to a dedicated L core (4,5,6,7)
64+
case $wifi_count in
65+
0) default_core=4 ;; # First WiFi device -> L core 4
66+
1) default_core=5 ;; # Second WiFi device -> L core 5
67+
2) default_core=6 ;; # Third WiFi device -> L core 6
68+
3) default_core=7 ;; # Fourth WiFi device -> L core 7
69+
esac
70+
71+
if [[ -e "/proc/irq/$irq/smp_affinity_list" ]]; then
72+
echo "$default_core" > "/proc/irq/$irq/smp_affinity_list"
73+
echo "WiFi default queue IRQ $irq -> CPU $default_core"
74+
else
75+
echo "Warning: WiFi default queue IRQ $irq not found"
76+
fi
77+
78+
# Set all other queues for this WiFi device to same core as default queue
79+
pci_device=$(${pkgs.gnugrep}/bin/grep "^ *$irq:" /proc/interrupts | ${pkgs.gnugrep}/bin/grep -o "0000:[0-9a-f:]*")
80+
if [ -n "$pci_device" ]; then
81+
for queue_irq in $(cat /proc/interrupts | ${pkgs.gnugrep}/bin/grep "$pci_device" | ${pkgs.gnugrep}/bin/grep -v "default_queue" | ${pkgs.gawk}/bin/awk '{print $1}' | ${pkgs.gnused}/bin/sed 's/://'); do
82+
if [[ -e "/proc/irq/$queue_irq/smp_affinity_list" ]]; then
83+
echo "$default_core" > "/proc/irq/$queue_irq/smp_affinity_list"
84+
echo " Queue IRQ $queue_irq -> CPU $default_core"
85+
fi
86+
done
87+
fi
88+
89+
wifi_count=$((wifi_count + 1))
90+
done
91+
else
92+
echo "Warning: No WiFi default queue IRQs found"
93+
fi
94+
3195
echo "IRQ affinity configuration complete"
3296
echo "Current IRQ distribution:"
33-
cat /proc/interrupts | grep -E "(enp|wlp)" | head -20
97+
cat /proc/interrupts | ${pkgs.gnugrep}/bin/grep -E "(enp|iwlwifi)" || true
3498
'';
3599

36100
in {

desktop/l2/irq-slice-analysis.sh

Lines changed: 217 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,217 @@
1+
#!/bin/bash
2+
3+
# IRQ and Slice Analysis Script for L2 WiFi Access Point
4+
# Analyzes runtime state of IRQ distribution and systemd slice CPU affinity
5+
6+
set -euo pipefail
7+
8+
echo "=== L2 WiFi Access Point - IRQ and Slice Analysis ==="
9+
echo "Generated: $(date)"
10+
echo ""
11+
12+
# Color codes for output
13+
RED='\033[0;31m'
14+
GREEN='\033[0;32m'
15+
YELLOW='\033[1;33m'
16+
BLUE='\033[0;34m'
17+
NC='\033[0m' # No Color
18+
19+
# Function to print colored output
20+
print_status() {
21+
local status=$1
22+
local message=$2
23+
case $status in
24+
"OK") echo -e "${GREEN}${NC} $message" ;;
25+
"WARN") echo -e "${YELLOW}${NC} $message" ;;
26+
"ERROR") echo -e "${RED}${NC} $message" ;;
27+
"INFO") echo -e "${BLUE}${NC} $message" ;;
28+
esac
29+
}
30+
31+
# Check kernel parameters
32+
cmdline=$(cat /proc/cmdline)
33+
isolcpus=$(echo "$cmdline" | grep -o "isolcpus=[^ ]*" | cut -d= -f2 || echo "NOT SET")
34+
print_status "INFO" "Isolated cores: $isolcpus"
35+
36+
# Ethernet IRQ L core distribution
37+
echo ""
38+
echo "=== Ethernet IRQ L Cores ==="
39+
enp1s0_irqs=$(cat /proc/interrupts | grep "enp1s0" | awk '{print $1}' | sed 's/://')
40+
if [ -n "$enp1s0_irqs" ]; then
41+
l_cores=""
42+
for irq in $enp1s0_irqs; do
43+
cpu_dist=$(grep "^ *$irq:" /proc/interrupts | awk '{for(i=2; i<=25; i++) if($i>0) printf "%d ", i-1}')
44+
for cpu in $cpu_dist; do
45+
if [ "$cpu" -lt 12 ]; then
46+
l_core=$cpu
47+
else
48+
l_core=$((cpu - 12))
49+
fi
50+
l_cores="$l_cores $l_core"
51+
done
52+
done
53+
l_cores=$(echo $l_cores | tr ' ' '\n' | sort -u | tr '\n' ' ')
54+
echo "enp1s0 IRQs using L cores: $l_cores"
55+
else
56+
print_status "WARN" "No enp1s0 IRQs found"
57+
fi
58+
59+
# WiFi IRQ L core distribution
60+
echo ""
61+
echo "=== WiFi IRQ L Cores ==="
62+
iwlwifi_irqs=$(cat /proc/interrupts | grep "iwlwifi" | awk '{print $1}' | sed 's/://')
63+
if [ -n "$iwlwifi_irqs" ]; then
64+
l_cores=""
65+
for irq in $iwlwifi_irqs; do
66+
cpu_dist=$(grep "^ *$irq:" /proc/interrupts | awk '{for(i=2; i<=25; i++) if($i>0) printf "%d ", i-1}')
67+
for cpu in $cpu_dist; do
68+
if [ "$cpu" -lt 12 ]; then
69+
l_core=$cpu
70+
else
71+
l_core=$((cpu - 12))
72+
fi
73+
l_cores="$l_cores $l_core"
74+
done
75+
done
76+
l_cores=$(echo $l_cores | tr ' ' '\n' | sort -u | tr '\n' ' ')
77+
echo "iwlwifi IRQs using L cores: $l_cores"
78+
79+
# Check expected cores 4,5,6,7
80+
for expected in 4 5 6 7; do
81+
if echo "$l_cores" | grep -q " $expected "; then
82+
print_status "OK" "L core $expected: used"
83+
else
84+
print_status "WARN" "L core $expected: not used"
85+
fi
86+
done
87+
else
88+
print_status "WARN" "No iwlwifi IRQs found"
89+
fi
90+
91+
# Systemd slice CPU affinity
92+
echo ""
93+
echo "=== Systemd Slice L Cores ==="
94+
95+
# Check global systemd CPU affinity
96+
echo "--- Global Systemd CPU Affinity ---"
97+
if [ -f "/etc/systemd/system.conf" ]; then
98+
global_cpu_affinity=$(grep "^CPUAffinity=" /etc/systemd/system.conf | cut -d= -f2 || echo "not set")
99+
echo "Global systemd CPUAffinity: $global_cpu_affinity"
100+
else
101+
echo "Global systemd CPUAffinity: not configured"
102+
fi
103+
104+
# Check main slices
105+
main_slices=("network-services" "system")
106+
for slice in "${main_slices[@]}"; do
107+
echo "--- $slice.slice ---"
108+
cgroup_path="/sys/fs/cgroup/system.slice/$slice.slice"
109+
110+
if systemctl status "$slice.slice" >/dev/null 2>&1; then
111+
if [ -d "$cgroup_path" ] && [ -f "$cgroup_path/cpuset.cpus" ]; then
112+
cpu_affinity=$(cat "$cgroup_path/cpuset.cpus")
113+
print_status "INFO" "CPU affinity: $cpu_affinity"
114+
115+
# Convert to L cores
116+
l_cores=""
117+
for cpu in $(echo $cpu_affinity | tr ',' ' '); do
118+
if [[ $cpu =~ ^[0-9]+$ ]]; then
119+
if [ "$cpu" -lt 12 ]; then
120+
l_core=$cpu
121+
else
122+
l_core=$((cpu - 12))
123+
fi
124+
l_cores="$l_cores $l_core"
125+
elif [[ $cpu =~ ^([0-9]+)-([0-9]+)$ ]]; then
126+
start=${BASH_REMATCH[1]}
127+
end=${BASH_REMATCH[2]}
128+
for ((cpu=start; cpu<=end; cpu++)); do
129+
if [ "$cpu" -lt 12 ]; then
130+
l_core=$cpu
131+
else
132+
l_core=$((cpu - 12))
133+
fi
134+
l_cores="$l_cores $l_core"
135+
done
136+
fi
137+
done
138+
l_cores=$(echo $l_cores | tr ' ' '\n' | sort -u | tr '\n' ' ')
139+
echo " L cores: $l_cores"
140+
else
141+
print_status "INFO" "Slice loaded but no cgroup (no active services)"
142+
fi
143+
else
144+
print_status "ERROR" "Slice $slice.slice not found or not active"
145+
fi
146+
echo ""
147+
done
148+
149+
# Check per-daemon slices
150+
per_daemon_slices=("kea" "pdns" "radvd" "hostapd")
151+
for slice in "${per_daemon_slices[@]}"; do
152+
echo "--- $slice.slice ---"
153+
cgroup_path="/sys/fs/cgroup/system.slice/$slice.slice"
154+
155+
if systemctl status "$slice.slice" >/dev/null 2>&1; then
156+
if [ -d "$cgroup_path" ] && [ -f "$cgroup_path/cpuset.cpus" ]; then
157+
cpu_affinity=$(cat "$cgroup_path/cpuset.cpus")
158+
print_status "INFO" "CPU affinity: $cpu_affinity"
159+
160+
# Convert to L cores
161+
l_cores=""
162+
for cpu in $(echo $cpu_affinity | tr ',' ' '); do
163+
if [[ $cpu =~ ^[0-9]+$ ]]; then
164+
if [ "$cpu" -lt 12 ]; then
165+
l_core=$cpu
166+
else
167+
l_core=$((cpu - 12))
168+
fi
169+
l_cores="$l_cores $l_core"
170+
elif [[ $cpu =~ ^([0-9]+)-([0-9]+)$ ]]; then
171+
start=${BASH_REMATCH[1]}
172+
end=${BASH_REMATCH[2]}
173+
for ((cpu=start; cpu<=end; cpu++)); do
174+
if [ "$cpu" -lt 12 ]; then
175+
l_core=$cpu
176+
else
177+
l_core=$((cpu - 12))
178+
fi
179+
l_cores="$l_cores $l_core"
180+
done
181+
fi
182+
done
183+
l_cores=$(echo $l_cores | tr ' ' '\n' | sort -u | tr '\n' ' ')
184+
echo " L cores: $l_cores"
185+
else
186+
print_status "INFO" "Slice loaded but no cgroup (no active services)"
187+
fi
188+
else
189+
print_status "WARN" "Slice $slice.slice not found or not active"
190+
fi
191+
echo ""
192+
done
193+
194+
# Service status summary
195+
echo ""
196+
echo "=== Service Status ==="
197+
services=("hostapd" "kea-dhcp4-server" "pdns-recursor" "radvd")
198+
for service in "${services[@]}"; do
199+
if systemctl is-active "$service" >/dev/null 2>&1; then
200+
slice=$(systemctl show "$service" --property=Slice --value 2>/dev/null || echo "unknown")
201+
nice_value=$(systemctl show "$service" --property=Nice --value 2>/dev/null || echo "0")
202+
print_status "OK" "$service: active (slice: $slice, nice: $nice_value)"
203+
else
204+
print_status "WARN" "$service: not active"
205+
fi
206+
done
207+
208+
# IRQ affinity service
209+
echo ""
210+
if systemctl is-active irq-affinity >/dev/null 2>&1; then
211+
print_status "OK" "IRQ affinity service: active"
212+
else
213+
print_status "WARN" "IRQ affinity service: not active"
214+
fi
215+
216+
echo ""
217+
echo "=== Analysis Complete ==="

0 commit comments

Comments
 (0)