Skip to content

Commit 5aa94c7

Browse files
XeartyXearty
committed
wip
1 parent 1fc04f4 commit 5aa94c7

File tree

3 files changed

+60
-52
lines changed

3 files changed

+60
-52
lines changed

‎packages/mcl/src/src/mcl/commands/ci.d‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,9 @@ import mcl.utils.process : execute;
1616
import mcl.utils.nix : nix;
1717
import mcl.utils.json : toJSON;
1818

19-
Params params;
20-
2119
export void ci()
2220
{
23-
params = parseEnv!Params;
21+
Params params = parseEnv!Params;
2422

2523
auto shardMatrix = generateShardMatrix();
2624
foreach (shard; shardMatrix.include)
@@ -53,7 +51,7 @@ export void ci()
5351
}
5452

5553
auto matrix = flakeAttr(params.flakePre, arch, os, params.flakePost)
56-
.nixEvalJobs(cachixUrl, false);
54+
.nixEvalJobs(params, cachixUrl, false);
5755

5856
foreach (pkg; matrix)
5957
{

‎packages/mcl/src/src/mcl/commands/ci_matrix.d‎

Lines changed: 53 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -14,14 +14,15 @@ import std.path : buildPath;
1414
import std.process : pipeProcess, wait, Redirect, kill;
1515
import std.exception : enforce;
1616
import std.format : fmt = format;
17-
import std.logger : tracef, infof, errorf;
17+
import std.logger : tracef, infof, errorf, warningf;
1818

1919
import mcl.utils.env : optional, MissingEnvVarsException, parseEnv;
2020
import mcl.utils.string : enumToString, StringRepresentation, MaxWidth, writeRecordAsTable;
2121
import mcl.utils.json : toJSON;
2222
import mcl.utils.path : rootDir, resultDir, gcRootsDir, createResultDirs;
2323
import mcl.utils.process : execute;
2424
import mcl.utils.nix : nix;
25+
import mcl.utils.cachix : cachixNixStoreUrl;
2526

2627
enum GitHubOS
2728
{
@@ -141,18 +142,12 @@ version (unittest)
141142
];
142143
}
143144

144-
immutable Params params;
145-
146-
version (unittest) {} else
147-
shared static this()
148-
{
149-
params = parseEnv!Params;
150-
}
151-
152145
export void ci_matrix()
153146
{
147+
Params params = parseEnv!Params;
148+
154149
createResultDirs();
155-
nixEvalForAllSystems().array.printTableForCacheStatus();
150+
nixEvalForAllSystems(params).array.printTableForCacheStatus(params);
156151
}
157152

158153
string flakeAttr(string prefix, SupportedSystem system, string postfix)
@@ -167,14 +162,14 @@ string flakeAttr(string prefix, string arch, string os, string postfix)
167162
return "%s.%s-%s%s".fmt(prefix, arch, os, postfix);
168163
}
169164

170-
Package[] checkCacheStatus(Package[] packages)
165+
Package[] checkCacheStatus(Package[] packages, string cachixAuthToken)
171166
{
172167
import std.array : appender;
173168
import std.parallelism : parallel;
174169

175170
foreach (ref pkg; packages.parallel)
176171
{
177-
pkg = checkPackage(pkg);
172+
pkg = checkPackage(pkg, cachixAuthToken);
178173
struct Output { string isCached, name, storePath; }
179174
auto res = appender!string;
180175
writeRecordAsTable(
@@ -188,11 +183,13 @@ Package[] checkCacheStatus(Package[] packages)
188183

189184
export void print_table()
190185
{
186+
Params params = parseEnv!Params;
187+
191188
createResultDirs();
192189

193-
getPrecalcMatrix()
194-
.checkCacheStatus()
195-
.printTableForCacheStatus();
190+
getPrecalcMatrix(params)
191+
.checkCacheStatus(params.cachixAuthToken)
192+
.printTableForCacheStatus(params);
196193
}
197194

198195
struct Params
@@ -304,12 +301,12 @@ unittest
304301
}
305302
}
306303

307-
Package[] nixEvalJobs(string flakeAttrPrefix, string cachixUrl, bool doCheck = true)
304+
Package[] nixEvalJobs(string flakeAttrPrefix, Params params, string cachixUrl, bool doCheck = true)
308305
{
309306
Package[] result = [];
310307

311-
int maxMemoryMB = getAvailableMemoryMB();
312-
int maxWorkers = getNixEvalWorkerCount();
308+
int maxMemoryMB = getAvailableMemoryMB(params.maxMemory);
309+
int maxWorkers = getNixEvalWorkerCount(params.maxWorkers);
313310

314311
const args = [
315312
"nix-eval-jobs", "--quiet", "--option", "warn-dirty", "false",
@@ -322,32 +319,34 @@ Package[] nixEvalJobs(string flakeAttrPrefix, string cachixUrl, bool doCheck = t
322319

323320
auto pipes = pipeProcess(args, Redirect.stdout | Redirect.stderr);
324321

322+
void logWarning(string errorMsg)
323+
{
324+
warningf("Command `%s` stderr:\n---\n%s\n---", args, errorMsg);
325+
}
326+
325327
void logError(string errorMsg)
326328
{
327329
errorf("Command `%s` failed with error:\n---\n%s\n---",
328330
args, errorMsg);
329331
}
330332

333+
bool jobFailed = false;
334+
331335
foreach (line; pipes.stdout.byLine)
332336
{
333-
if (line.indexOf("{") == -1)
334-
{
335-
errorf("Expected JSON object on stdout from nix-eval-jobs, got: `%s`", line);
336-
continue;
337-
}
338-
339337
auto json = parseJSON(line);
340338

341339
if (auto err = "error" in json)
342340
{
343341
logError((*err).str);
342+
jobFailed = true;
344343
continue; // drain the output
345344
}
346345

347346
Package pkg = json.packageFromNixEvalJobsJson(
348347
flakeAttrPrefix, cachixUrl);
349348

350-
if (doCheck)pkg = pkg.checkPackage();
349+
if (doCheck)pkg = pkg.checkPackage(params.cachixAuthToken);
351350

352351
result ~= pkg;
353352

@@ -365,16 +364,20 @@ Package[] nixEvalJobs(string flakeAttrPrefix, string cachixUrl, bool doCheck = t
365364
output: pkg.output
366365
).writeRecordAsTable(stderr.lockingTextWriter);
367366
}
367+
368+
string bufferedOutput = "";
368369
foreach (line; pipes.stderr.byLine)
369370
{
370371
if (uselessWarnings.map!((warning) => line.indexOf(warning) != -1).any)
371372
continue;
372373

373-
logError(line.idup);
374+
bufferedOutput ~= line ~ "\n";
374375
}
375376

377+
logWarning(bufferedOutput);
378+
376379
int status = wait(pipes.pid);
377-
enforce(status == 0, "Command `%s` failed with status %s".fmt(args, status));
380+
enforce(status == 0 && !jobFailed, "Command `%s` failed with status %s".fmt(args, status));
378381

379382
return result;
380383
}
@@ -396,26 +399,26 @@ SupportedSystem[] getSupportedSystems(string flakeRef = ".")
396399
return json.array.map!(system => getSystem(system.str)).array;
397400
}
398401

399-
Package[] nixEvalForAllSystems()
402+
Package[] nixEvalForAllSystems(Params params)
400403
{
401-
const cachixUrl = "https://" ~ params.cachixCache ~ ".cachix.org";
404+
const cachixUrl = cachixNixStoreUrl(params.cachixCache);
402405
const systems = getSupportedSystems();
403406

404407
infof("Evaluating flake for: %s", systems);
405408

406409
return systems.map!(system =>
407410
flakeAttr(params.flakePre, system, params.flakePost)
408-
.nixEvalJobs(cachixUrl)
411+
.nixEvalJobs(params, cachixUrl)
409412
)
410413
.reduce!((a, b) => a ~ b)
411414
.array
412415
.sort!((a, b) => a.name < b.name)
413416
.array;
414417
}
415418

416-
int getNixEvalWorkerCount()
419+
int getNixEvalWorkerCount(int maxWorkers)
417420
{
418-
return params.maxWorkers == 0 ? (threadsPerCPU() < 8 ? threadsPerCPU() : 8) : params.maxWorkers;
421+
return maxWorkers == 0 ? (threadsPerCPU() < 8 ? threadsPerCPU() : 8) : maxWorkers;
419422
}
420423

421424
@("getNixEvalWorkerCount")
@@ -424,7 +427,7 @@ unittest
424427
assert(getNixEvalWorkerCount() == (threadsPerCPU() < 8 ? threadsPerCPU() : 8));
425428
}
426429

427-
int getAvailableMemoryMB()
430+
int getAvailableMemoryMB(int maxMemory)
428431
{
429432

430433
// free="$(< /proc/meminfo grep MemFree | tr -s ' ' | cut -d ' ' -f 2)"
@@ -448,15 +451,15 @@ int getAvailableMemoryMB()
448451
.find!(a => a.indexOf("Shmem:") != -1)
449452
.front
450453
.split[1].to!int;
451-
int maxMemoryMB = params.maxMemory == 0 ? ((free + cached + buffers + shmem) / 1024)
452-
: params.maxMemory;
454+
int maxMemoryMB = maxMemory == 0 ? ((free + cached + buffers + shmem) / 1024) : maxMemory;
453455
return maxMemoryMB;
454456
}
455457

456458
@("getAvailableMemoryMB")
457459
unittest
458460
{
459-
assert(getAvailableMemoryMB() > 0);
461+
Params params = parseEnv!Params;
462+
assert(getAvailableMemoryMB(params.maxMemory) > 0);
460463
}
461464

462465
void saveCachixDeploySpec(Package[] packages)
@@ -481,12 +484,12 @@ unittest
481484
assert(testPackageArray[1].output == deploySpec[0]["out"].str);
482485
}
483486

484-
void saveGHCIMatrix(Package[] packages)
487+
void saveGHCIMatrix(Package[] packages, bool isInitial)
485488
{
486489
auto matrix = JSONValue([
487490
"include": JSONValue(packages.map!(pkg => pkg.toJSON()).array)
488491
]);
489-
string resPath = rootDir.buildPath(params.isInitial ? "matrix-pre.json" : "matrix-post.json");
492+
string resPath = rootDir.buildPath(isInitial ? "matrix-pre.json" : "matrix-post.json");
490493
resPath.write(JSONValue(matrix).toString(JSONOptions.doNotEscapeSlashes));
491494
}
492495

@@ -495,8 +498,10 @@ unittest
495498
{
496499
import std.file : rmdirRecurse;
497500

501+
Params params = parseEnv!Params;
502+
498503
createResultDirs();
499-
saveGHCIMatrix(cast(Package[]) testPackageArray);
504+
saveGHCIMatrix(cast(Package[]) testPackageArray, params.isInitial);
500505
JSONValue matrix = rootDir
501506
.buildPath(params.isInitial ? "matrix-pre.json" : "matrix-post.json")
502507
.readText
@@ -624,24 +629,24 @@ unittest
624629

625630
}
626631

627-
void printTableForCacheStatus(Package[] packages)
632+
void printTableForCacheStatus(Package[] packages, Params params)
628633
{
629634
if (params.precalcMatrix == "")
630635
{
631-
saveGHCIMatrix(packages);
636+
saveGHCIMatrix(packages, params.isInitial);
632637
}
633638
saveCachixDeploySpec(packages);
634639
saveGHCIComment(convertNixEvalToTableSummary(packages, params.isInitial));
635640
}
636641

637-
Package checkPackage(Package pkg)
642+
Package checkPackage(Package pkg, ref string cachixAuthToken)
638643
{
639644
import std.algorithm : canFind;
640645
import std.string : lineSplitter;
641646
import std.net.curl : HTTP, httpGet = get, HTTPStatusException;
642647

643648
auto http = HTTP();
644-
http.addRequestHeader("Authorization", "Bearer " ~ params.cachixAuthToken);
649+
http.addRequestHeader("Authorization", "Bearer " ~ cachixAuthToken);
645650

646651
try
647652
{
@@ -663,6 +668,8 @@ Package checkPackage(Package pkg)
663668
@("checkPackage")
664669
unittest
665670
{
671+
Params params = parseEnv!Params;
672+
666673
const nixosCacheEndpoint = "https://cache.nixos.org/";
667674
const storePathHash = "mdb034kf7sq6g03ric56jxr4a7043l41";
668675
const storePath = "/nix/store/" ~ storePathHash ~ "-hello-2.12.1";
@@ -673,14 +680,14 @@ unittest
673680
);
674681

675682
assert(!testPackage.isCached);
676-
assert(checkPackage(testPackage).isCached);
683+
assert(checkPackage(testPackage, params.cachixAuthToken).isCached);
677684

678685
testPackage.cacheUrl = nixosCacheEndpoint ~ "nonexistent.narinfo";
679686

680-
assert(!checkPackage(testPackage).isCached);
687+
assert(!checkPackage(testPackage, params.cachixAuthToken).isCached);
681688
}
682689

683-
Package[] getPrecalcMatrix()
690+
Package[] getPrecalcMatrix(Params params)
684691
{
685692
auto precalcMatrixStr = params.precalcMatrix == "" ? "{\"include\": []}" : params.precalcMatrix;
686693
enforce!MissingEnvVarsException(

‎packages/mcl/src/src/mcl/commands/deploy_spec.d‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,16 +14,19 @@ import mcl.utils.cachix : cachixNixStoreUrl, DeploySpec, createMachineDeploySpec
1414
import mcl.utils.tui : bold;
1515
import mcl.utils.json : toJSON, fromJSON, tryDeserializeFromJsonFile;
1616

17-
import mcl.commands.ci_matrix : flakeAttr, params, Params, nixEvalJobs, SupportedSystem;
17+
import mcl.commands.ci_matrix : flakeAttr, Params, nixEvalJobs, SupportedSystem;
1818

1919
export void deploy_spec()
2020
{
21+
2122
const deploySpecFile = resultDir.buildPath("cachix-deploy-spec.json");
2223

2324
if (!exists(deploySpecFile))
2425
{
26+
Params params = parseEnv!Params;
27+
2528
auto nixosConfigs = flakeAttr("legacyPackages", SupportedSystem.x86_64_linux, "bareMetalMachines")
26-
.nixEvalJobs(params.cachixCache.cachixNixStoreUrl);
29+
.nixEvalJobs(params, params.cachixCache.cachixNixStoreUrl);
2730

2831
auto configsMissingFromCachix = nixosConfigs.filter!(c => !c.isCached);
2932

0 commit comments

Comments
 (0)