Skip to content

Commit e676bbe

Browse files
Conni2461Conni2461
committed
feat: init hydra-queue-runner-v2
1 parent c2b4f36 commit e676bbe

File tree

13 files changed

+994
-46
lines changed

13 files changed

+994
-46
lines changed

‎dns/nixos.org.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@ D("nixos.org",
9696
// hydra staging area
9797
A("staging-hydra", "157.180.25.203"),
9898
AAAA("staging-hydra", "2a01:4f9:c012:d5d3::1"),
99+
CNAME("queue-runner.staging-hydra", "staging-hydra"),
99100

100101
// services infra
101102
A("caliban", "65.109.26.213"),

‎non-critical-infra/hosts/staging-hydra/ca.crt‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
-----BEGIN CERTIFICATE-----
2+
MIIBnTCCAU+gAwIBAgIUQpxYsPwAyTY70yYO9fcCmCaZreIwBQYDK2VwMEMxCzAJ
3+
BgNVBAYTAkRFMRQwEgYDVQQKDAtOaXhPUyBJbmZyYTEeMBwGA1UEAwwVaHlkcmEt
4+
cXVldWUtcnVubmVyLWNhMCAXDTI1MDczMTEyMDQxN1oYDzIwNzUwNzE5MTIwNDE3
5+
WjBDMQswCQYDVQQGEwJERTEUMBIGA1UECgwLTml4T1MgSW5mcmExHjAcBgNVBAMM
6+
FWh5ZHJhLXF1ZXVlLXJ1bm5lci1jYTAqMAUGAytlcAMhAM+Mc/XSTXwJeWPxrpqo
7+
SPT5Xwi8/j85VO6TsfBlXFt4o1MwUTAdBgNVHQ4EFgQU0wQG6BxTKtYwlywuyD0a
8+
Vr/1r4gwHwYDVR0jBBgwFoAU0wQG6BxTKtYwlywuyD0aVr/1r4gwDwYDVR0TAQH/
9+
BAUwAwEB/zAFBgMrZXADQQA3BRP2+TkkDQPnPy6MQyDCxqfEeV6OQjtspSvCO0UL
10+
GWmfvzrlUQytwTFTPfVzaErbyVPbeYU5y8rmRoGPNSoI
11+
-----END CERTIFICATE-----

‎non-critical-infra/hosts/staging-hydra/client.crt‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
-----BEGIN CERTIFICATE-----
2+
MIIBlDCCAUagAwIBAgIUfUYjDOaJML1lIMkAMvLjnSTscYwwBQYDK2VwMEMxCzAJ
3+
BgNVBAYTAkRFMRQwEgYDVQQKDAtOaXhPUyBJbmZyYTEeMBwGA1UEAwwVaHlkcmEt
4+
cXVldWUtcnVubmVyLWNhMCAXDTI1MDczMTEyMDQxN1oYDzIwNzUwNzE5MTIwNDE3
5+
WjBLMQswCQYDVQQGEwJERTEUMBIGA1UECgwLTml4T1MgSW5mcmExJjAkBgNVBAMM
6+
HWh5ZHJhLXF1ZXVlLWJ1aWxkZXItbG9jYWxob3N0MCowBQYDK2VwAyEAJqCvjdwY
7+
0l4z0cbwecONx3DfhpPo+SBymv7H2OQt4aejQjBAMB0GA1UdDgQWBBQw54bw+CrK
8+
O53J4qzB4Cknzf7CzTAfBgNVHSMEGDAWgBTTBAboHFMq1jCXLC7IPRpWv/WviDAF
9+
BgMrZXADQQB+XUCZQo0aX6IyEQDrds1d3SWF1rFBv5MjtZS25TYpnQQPgt9iiD6Q
10+
dtCwSVqyAgCG4XlkYBpU4AKVKqyMsSAC
11+
-----END CERTIFICATE-----

‎non-critical-infra/hosts/staging-hydra/default.nix‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55
inputs.srvos.nixosModules.server
66
inputs.srvos.nixosModules.hardware-hetzner-cloud-arm
77
../../modules/common.nix
8+
../../modules/hydra-queue-runner-v2.nix
9+
../../modules/hydra-queue-builder-v2.nix
810
./hydra-proxy.nix
911
./hydra.nix
1012
inputs.hydra.nixosModules.hydra

‎non-critical-infra/hosts/staging-hydra/genca.sh‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
#!/usr/bin/env bash
2+
3+
set -x
4+
5+
hosts="localhost ofborg-eval02 ofborg-eval03 ofborg-eval04 ofborg-build01 ofborg-build02 ofborg-build03 ofborg-build04 ofborg-build05"
6+
7+
C="DE"
8+
O="NixOS Infra"
9+
10+
newDir="$(date '+%Y-%m-%dT%H:%M')"
11+
mkdir "${newDir}"
12+
cd "${newDir}" || exit
13+
14+
openssl genpkey -algorithm Ed25519 -out ca.key
15+
openssl req -x509 -new -nodes -key ca.key -sha256 -days 18250 -out ca.crt \
16+
-subj "/C=${C}/O=${O}/CN=hydra-queue-runner-ca"
17+
18+
cat <<EOF >server.cnf
19+
[req]
20+
prompt = no
21+
x509_extensions = v3_req
22+
req_extensions = v3_req
23+
default_md = sha256
24+
distinguished_name = req_distinguished_name
25+
26+
[req_distinguished_name]
27+
C = ${C}
28+
O = ${O}
29+
CN = queue-runner.staging-hydra.nixos.org
30+
31+
[v3_req]
32+
basicConstraints = CA:FALSE
33+
keyUsage = nonRepudiation, digitalSignature, keyEncipherment, keyAgreement
34+
extendedKeyUsage = critical, serverAuth
35+
subjectAltName = @alt_names
36+
37+
[alt_names]
38+
DNS.1 = queue-runner.staging-hydra.nixos.org
39+
EOF
40+
41+
openssl genpkey -algorithm Ed25519 -out server.key
42+
openssl req -new -key server.key -out server.csr -config server.cnf
43+
openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out server.crt -days 18250 -sha256 -extfile server.cnf -extensions v3_req
44+
45+
for host in ${hosts}; do
46+
openssl genpkey -algorithm Ed25519 -out "client-${host}.key"
47+
openssl req -new -key "client-${host}.key" -out "client-${host}.csr" \
48+
-subj "/C=${C}/O=${O}/CN=hydra-queue-builder-${host}"
49+
openssl x509 -req -in "client-${host}.csr" -CA ca.crt -CAkey ca.key -CAcreateserial -out "client-${host}.crt" -days 18250 -sha256
50+
done
51+
52+
rm -rf -- *.csr *.srl
53+
rm server.cnf
54+
55+
cd - || exit

‎non-critical-infra/hosts/staging-hydra/hydra.nix‎

Lines changed: 117 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -44,67 +44,136 @@ in
4444
};
4545
};
4646

47-
services.hydra-dev = {
48-
enable = true;
49-
package = pkgs.hydra;
50-
buildMachinesFiles = [
51-
(pkgs.writeText "local" ''
52-
localhost ${lib.concatStringsSep "," localSystems} - 3 1 ${lib.concatStringsSep "," config.nix.settings.system-features} - -
53-
'')
54-
];
55-
logo = ../../../build/hydra-logo.png;
56-
hydraURL = "https://hydra.nixos.org";
57-
notificationSender = "[email protected]";
58-
smtpHost = "localhost";
59-
useSubstitutes = true;
60-
extraConfig = ''
61-
max_servers 30
47+
services = {
48+
hydra-dev = {
49+
enable = true;
50+
package = pkgs.hydra;
51+
buildMachinesFiles = [
52+
(pkgs.writeText "local" ''
53+
localhost ${lib.concatStringsSep "," localSystems} - 3 1 ${lib.concatStringsSep "," config.nix.settings.system-features} - -
54+
'')
55+
];
56+
logo = ../../../build/hydra-logo.png;
57+
hydraURL = "https://hydra.nixos.org";
58+
notificationSender = "[email protected]";
59+
smtpHost = "localhost";
60+
useSubstitutes = true;
61+
extraConfig = ''
62+
max_servers 30
6263
63-
store_uri = s3://nix-cache-staging?secret-key=${config.sops.secrets.signing-key.path}&ls-compression=br&log-compression=br
64-
server_store_uri = https://cache-staging.nixos.org?local-nar-cache=${narCache}
65-
binary_cache_public_uri = https://cache-staging.nixos.org
64+
store_uri = s3://nix-cache-staging?secret-key=${config.sops.secrets.signing-key.path}&ls-compression=br&log-compression=br
65+
server_store_uri = https://cache-staging.nixos.org?local-nar-cache=${narCache}
66+
binary_cache_public_uri = https://cache-staging.nixos.org
6667
67-
<Plugin::Session>
68-
cache_size = 32m
69-
</Plugin::Session>
68+
<Plugin::Session>
69+
cache_size = 32m
70+
</Plugin::Session>
7071
71-
# patchelf:master:3
72-
xxx-jobset-repeats = nixos:reproducibility:1
72+
# patchelf:master:3
73+
xxx-jobset-repeats = nixos:reproducibility:1
7374
74-
upload_logs_to_binary_cache = true
75-
compress_build_logs = false # conflicts with upload_logs_to_binary_cache
75+
upload_logs_to_binary_cache = true
76+
compress_build_logs = false # conflicts with upload_logs_to_binary_cache
7677
77-
log_prefix = https://cache.nixos.org/
78+
log_prefix = https://cache.nixos.org/
7879
79-
evaluator_workers = 1
80-
evaluator_max_memory_size = 4096
80+
evaluator_workers = 1
81+
evaluator_max_memory_size = 4096
8182
82-
max_concurrent_evals = 1
83+
queue_runner_endpoint = http://localhost:8080
8384
84-
# increase the number of active compress slots (CPU is 48*2 on mimas)
85-
max_local_worker_threads = 144
85+
max_concurrent_evals = 1
8686
87-
max_unsupported_time = 86400
87+
# increase the number of active compress slots (CPU is 48*2 on mimas)
88+
max_local_worker_threads = 144
8889
89-
allow_import_from_derivation = false
90+
max_unsupported_time = 86400
9091
91-
max_output_size = 3821225472 # 3 << 30 + 600000000 = 3 GiB + 0.6 GB
92-
max_db_connections = 350
92+
allow_import_from_derivation = false
9393
94-
queue_runner_metrics_address = [::]:9198
94+
max_output_size = 3821225472 # 3 << 30 + 600000000 = 3 GiB + 0.6 GB
95+
max_db_connections = 350
9596
96-
<hydra_notify>
97-
<prometheus>
98-
listen_address = 0.0.0.0
99-
port = 9199
100-
</prometheus>
101-
</hydra_notify>
102-
'';
97+
queue_runner_metrics_address = [::]:9198
98+
99+
<hydra_notify>
100+
<prometheus>
101+
listen_address = 0.0.0.0
102+
port = 9199
103+
</prometheus>
104+
</hydra_notify>
105+
'';
106+
};
107+
108+
hydra-queue-runner-v2 = {
109+
enable = true;
110+
settings.remoteStoreAddr = [
111+
"s3://nix-cache-staging?secret-key=${config.sops.secrets.signing-key.path}&ls-compression=br&log-compression=br"
112+
];
113+
};
114+
115+
hydra-queue-builder-v2 = {
116+
enable = true;
117+
queueRunnerAddr = "https://queue-runner.staging-hydra.nixos.org";
118+
mtls = {
119+
serverRootCaCertPath = "${./ca.crt}";
120+
clientCertPath = "${./client.crt}";
121+
clientKeyPath = config.sops.secrets."queue-runner-client.key".path;
122+
domainName = "queue-runner.staging-hydra.nixos.org";
123+
};
124+
};
125+
126+
nginx = {
127+
enable = true;
128+
virtualHosts."queue-runner.staging-hydra.nixos.org" = {
129+
extraConfig = ''
130+
ssl_client_certificate ${./ca.crt};
131+
ssl_verify_depth 2;
132+
ssl_verify_client on;
133+
'';
134+
135+
sslCertificate = ./server.crt;
136+
sslCertificateKey = config.sops.secrets."queue-runner-server.key".path;
137+
onlySSL = true;
138+
139+
locations."/".extraConfig = ''
140+
# This is necessary so that grpc connections do not get closed early
141+
# see https://stackoverflow.com/a/67805465
142+
client_body_timeout 31536000s;
143+
144+
grpc_pass grpc://[::1]:50051;
145+
146+
grpc_read_timeout 31536000s; # 1 year in seconds
147+
grpc_send_timeout 31536000s; # 1 year in seconds
148+
grpc_socket_keepalive on;
149+
150+
grpc_set_header Host $host;
151+
grpc_set_header X-Real-IP $remote_addr;
152+
grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
153+
grpc_set_header X-Forwarded-Proto $scheme;
154+
155+
grpc_set_header X-Client-DN $ssl_client_s_dn;
156+
grpc_set_header X-Client-Cert $ssl_client_escaped_cert;
157+
'';
158+
};
159+
};
103160
};
104161

105-
sops.secrets.hydra-users = {
106-
sopsFile = ../../secrets/hydra-users.staging-hydra;
107-
format = "binary";
162+
sops.secrets = {
163+
"queue-runner-server.key" = {
164+
sopsFile = ../../secrets/queue-runner-server.key.staging-hydra;
165+
format = "binary";
166+
owner = config.systemd.services.hydra-queue-runner.serviceConfig.User;
167+
};
168+
"queue-runner-client.key" = {
169+
sopsFile = ../../secrets/queue-runner-client.key.staging-hydra;
170+
format = "binary";
171+
owner = config.systemd.services.hydra-queue-runner.serviceConfig.User;
172+
};
173+
hydra-users = {
174+
sopsFile = ../../secrets/hydra-users.staging-hydra;
175+
format = "binary";
176+
};
108177
};
109178

110179
systemd = {
@@ -117,6 +186,8 @@ in
117186
services = {
118187
hydra-notify.enable = false;
119188
hydra-queue-runner = {
189+
enable = false;
190+
120191
# restarting the scheduler is very expensive
121192
restartIfChanged = false;
122193
serviceConfig = {

‎non-critical-infra/hosts/staging-hydra/server.crt‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
-----BEGIN CERTIFICATE-----
2+
MIIB/jCCAbCgAwIBAgIUfUYjDOaJML1lIMkAMvLjnSTscYswBQYDK2VwMEMxCzAJ
3+
BgNVBAYTAkRFMRQwEgYDVQQKDAtOaXhPUyBJbmZyYTEeMBwGA1UEAwwVaHlkcmEt
4+
cXVldWUtcnVubmVyLWNhMCAXDTI1MDczMTEyMDQxN1oYDzIwNzUwNzE5MTIwNDE3
5+
WjBSMQswCQYDVQQGEwJERTEUMBIGA1UECgwLTml4T1MgSW5mcmExLTArBgNVBAMM
6+
JHF1ZXVlLXJ1bm5lci5zdGFnaW5nLWh5ZHJhLm5peG9zLm9yZzAqMAUGAytlcAMh
7+
ANVnDi5rY0Ar4hPbqRJqS+Nw7b5GTg0QxL2DM7l1xTqHo4GkMIGhMAkGA1UdEwQC
8+
MAAwCwYDVR0PBAQDAgPoMBYGA1UdJQEB/wQMMAoGCCsGAQUFBwMBMC8GA1UdEQQo
9+
MCaCJHF1ZXVlLXJ1bm5lci5zdGFnaW5nLWh5ZHJhLm5peG9zLm9yZzAdBgNVHQ4E
10+
FgQU4ArR8rzVAt6dFkSXiMUlYYAzbwUwHwYDVR0jBBgwFoAU0wQG6BxTKtYwlywu
11+
yD0aVr/1r4gwBQYDK2VwA0EAScS72oaQ8PcYpH26FuRGnKaWe4e7fQ5RmKBUyC+5
12+
CiYIWu4D7fNGYJ15szCfh4nJIuyB0eXBv1ddAGAQMVdhDw==
13+
-----END CERTIFICATE-----

0 commit comments

Comments
 (0)