wip

Author: Xearty

packages/mcl/src/src/mcl/commands/ci.d

@@ -16,11 +16,9 @@ import mcl.utils.process : execute;
 import mcl.utils.nix : nix;
 import mcl.utils.json : toJSON;
 
-Params params;
-
 export void ci()
 {
-    params = parseEnv!Params;
+    Params params = parseEnv!Params;
 
     auto shardMatrix = generateShardMatrix();
     foreach (shard; shardMatrix.include)
@@ -53,7 +51,7 @@ export void ci()
         }
 
         auto matrix = flakeAttr(params.flakePre, arch, os, params.flakePost)
-            .nixEvalJobs(cachixUrl, false);
+            .nixEvalJobs(params, cachixUrl, false);
 
         foreach (pkg; matrix)
         {

packages/mcl/src/src/mcl/commands/ci_matrix.d

@@ -22,6 +22,7 @@ import mcl.utils.json : toJSON;
 import mcl.utils.path : rootDir, resultDir, gcRootsDir, createResultDirs;
 import mcl.utils.process : execute;
 import mcl.utils.nix : nix;
+import mcl.utils.cachix : cachixNixStoreUrl;
 
 enum GitHubOS
 {
@@ -141,18 +142,12 @@ version (unittest)
     ];
 }
 
-immutable Params params;
-
-version (unittest) {} else
-shared static this()
-{
-    params = parseEnv!Params;
-}
-
 export void ci_matrix()
 {
+    Params params = parseEnv!Params;
+
     createResultDirs();
-    nixEvalForAllSystems().array.printTableForCacheStatus();
+    nixEvalForAllSystems(params).array.printTableForCacheStatus(params);
 }
 
 string flakeAttr(string prefix, SupportedSystem system, string postfix)
@@ -167,14 +162,14 @@ string flakeAttr(string prefix, string arch, string os, string postfix)
     return "%s.%s-%s%s".fmt(prefix, arch, os, postfix);
 }
 
-Package[] checkCacheStatus(Package[] packages)
+Package[] checkCacheStatus(Package[] packages, string cachixAuthToken)
 {
     import std.array : appender;
     import std.parallelism : parallel;
 
     foreach (ref pkg; packages.parallel)
     {
-        pkg = checkPackage(pkg);
+        pkg = checkPackage(pkg, cachixAuthToken);
         struct Output { string isCached, name, storePath; }
         auto res = appender!string;
         writeRecordAsTable(
@@ -188,11 +183,13 @@ Package[] checkCacheStatus(Package[] packages)
 
 export void print_table()
 {
+    Params params = parseEnv!Params;
+
     createResultDirs();
 
-    getPrecalcMatrix()
-        .checkCacheStatus()
-        .printTableForCacheStatus();
+    getPrecalcMatrix(params)
+        .checkCacheStatus(params.cachixAuthToken)
+        .printTableForCacheStatus(params);
 }
 
 struct Params
@@ -304,12 +301,12 @@ unittest
     }
 }
 
-Package[] nixEvalJobs(string flakeAttrPrefix, string cachixUrl, bool doCheck = true)
+Package[] nixEvalJobs(string flakeAttrPrefix, Params params, string cachixUrl, bool doCheck = true)
 {
     Package[] result = [];
 
-    int maxMemoryMB = getAvailableMemoryMB();
-    int maxWorkers = getNixEvalWorkerCount();
+    int maxMemoryMB = getAvailableMemoryMB(params.maxMemory);
+    int maxWorkers = getNixEvalWorkerCount(params.maxWorkers);
 
     const args = [
         "nix-eval-jobs", "--quiet", "--option", "warn-dirty", "false",
@@ -347,7 +344,7 @@ Package[] nixEvalJobs(string flakeAttrPrefix, string cachixUrl, bool doCheck = t
         Package pkg = json.packageFromNixEvalJobsJson(
             flakeAttrPrefix, cachixUrl);
 
-        if (doCheck)pkg = pkg.checkPackage();
+        if (doCheck)pkg = pkg.checkPackage(params.cachixAuthToken);
 
         result ~= pkg;
 
@@ -365,14 +362,19 @@ Package[] nixEvalJobs(string flakeAttrPrefix, string cachixUrl, bool doCheck = t
             output: pkg.output
         ).writeRecordAsTable(stderr.lockingTextWriter);
     }
+
+    string bufferedOutput = "";
     foreach (line; pipes.stderr.byLine)
     {
         if (uselessWarnings.map!((warning) => line.indexOf(warning) != -1).any)
             continue;
 
-        logError(line.idup);
+        bufferedOutput ~= line ~ "\n";
+        // logError(line.idup);
     }
 
+    logError(bufferedOutput);
+
     int status = wait(pipes.pid);
     enforce(status == 0, "Command `%s` failed with status %s".fmt(args, status));
 
@@ -396,26 +398,26 @@ SupportedSystem[] getSupportedSystems(string flakeRef = ".")
     return json.array.map!(system => getSystem(system.str)).array;
 }
 
-Package[] nixEvalForAllSystems()
+Package[] nixEvalForAllSystems(Params params)
 {
-    const cachixUrl = "https://" ~ params.cachixCache ~ ".cachix.org";
+    const cachixUrl = cachixNixStoreUrl(params.cachixCache);
     const systems = getSupportedSystems();
 
     infof("Evaluating flake for: %s", systems);
 
     return systems.map!(system =>
             flakeAttr(params.flakePre, system, params.flakePost)
-                .nixEvalJobs(cachixUrl)
+                .nixEvalJobs(params, cachixUrl)
         )
         .reduce!((a, b) => a ~ b)
         .array
         .sort!((a, b) => a.name < b.name)
         .array;
 }
 
-int getNixEvalWorkerCount()
+int getNixEvalWorkerCount(int maxWorkers)
 {
-    return params.maxWorkers == 0 ? (threadsPerCPU() < 8 ? threadsPerCPU() : 8) : params.maxWorkers;
+    return maxWorkers == 0 ? (threadsPerCPU() < 8 ? threadsPerCPU() : 8) : maxWorkers;
 }
 
 @("getNixEvalWorkerCount")
@@ -424,7 +426,7 @@ unittest
     assert(getNixEvalWorkerCount() == (threadsPerCPU() < 8 ? threadsPerCPU() : 8));
 }
 
-int getAvailableMemoryMB()
+int getAvailableMemoryMB(int maxMemory)
 {
 
     // free="$(< /proc/meminfo grep MemFree | tr -s ' ' | cut -d ' ' -f 2)"
@@ -448,15 +450,15 @@ int getAvailableMemoryMB()
         .find!(a => a.indexOf("Shmem:") != -1)
         .front
         .split[1].to!int;
-    int maxMemoryMB = params.maxMemory == 0 ? ((free + cached + buffers + shmem) / 1024)
-        : params.maxMemory;
+    int maxMemoryMB = maxMemory == 0 ? ((free + cached + buffers + shmem) / 1024) : maxMemory;
     return maxMemoryMB;
 }
 
 @("getAvailableMemoryMB")
 unittest
 {
-    assert(getAvailableMemoryMB() > 0);
+    Params params = parseEnv!Params;
+    assert(getAvailableMemoryMB(params.maxMemory) > 0);
 }
 
 void saveCachixDeploySpec(Package[] packages)
@@ -481,12 +483,12 @@ unittest
     assert(testPackageArray[1].output == deploySpec[0]["out"].str);
 }
 
-void saveGHCIMatrix(Package[] packages)
+void saveGHCIMatrix(Package[] packages, bool isInitial)
 {
     auto matrix = JSONValue([
         "include": JSONValue(packages.map!(pkg => pkg.toJSON()).array)
     ]);
-    string resPath = rootDir.buildPath(params.isInitial ? "matrix-pre.json" : "matrix-post.json");
+    string resPath = rootDir.buildPath(isInitial ? "matrix-pre.json" : "matrix-post.json");
     resPath.write(JSONValue(matrix).toString(JSONOptions.doNotEscapeSlashes));
 }
 
@@ -495,8 +497,10 @@ unittest
 {
     import std.file : rmdirRecurse;
 
+    Params params = parseEnv!Params;
+
     createResultDirs();
-    saveGHCIMatrix(cast(Package[]) testPackageArray);
+    saveGHCIMatrix(cast(Package[]) testPackageArray, params.isInitial);
     JSONValue matrix = rootDir
         .buildPath(params.isInitial ? "matrix-pre.json" : "matrix-post.json")
         .readText
@@ -624,24 +628,24 @@ unittest
 
 }
 
-void printTableForCacheStatus(Package[] packages)
+void printTableForCacheStatus(Package[] packages, Params params)
 {
     if (params.precalcMatrix == "")
     {
-        saveGHCIMatrix(packages);
+        saveGHCIMatrix(packages, params.isInitial);
     }
     saveCachixDeploySpec(packages);
     saveGHCIComment(convertNixEvalToTableSummary(packages, params.isInitial));
 }
 
-Package checkPackage(Package pkg)
+Package checkPackage(Package pkg, ref string cachixAuthToken)
 {
     import std.algorithm : canFind;
     import std.string : lineSplitter;
     import std.net.curl : HTTP, httpGet = get, HTTPStatusException;
 
     auto http = HTTP();
-    http.addRequestHeader("Authorization", "Bearer " ~ params.cachixAuthToken);
+    http.addRequestHeader("Authorization", "Bearer " ~ cachixAuthToken);
 
     try
     {
@@ -663,6 +667,8 @@ Package checkPackage(Package pkg)
 @("checkPackage")
 unittest
 {
+    Params params = parseEnv!Params;
+
     const nixosCacheEndpoint = "https://cache.nixos.org/";
     const storePathHash = "mdb034kf7sq6g03ric56jxr4a7043l41";
     const storePath = "/nix/store/" ~ storePathHash ~ "-hello-2.12.1";
@@ -673,14 +679,14 @@ unittest
     );
 
     assert(!testPackage.isCached);
-    assert(checkPackage(testPackage).isCached);
+    assert(checkPackage(testPackage, params.cachixAuthToken).isCached);
 
     testPackage.cacheUrl = nixosCacheEndpoint ~ "nonexistent.narinfo";
 
-    assert(!checkPackage(testPackage).isCached);
+    assert(!checkPackage(testPackage, params.cachixAuthToken).isCached);
 }
 
-Package[] getPrecalcMatrix()
+Package[] getPrecalcMatrix(Params params)
 {
     auto precalcMatrixStr = params.precalcMatrix == "" ? "{\"include\": []}" : params.precalcMatrix;
     enforce!MissingEnvVarsException(

packages/mcl/src/src/mcl/commands/deploy_spec.d

@@ -14,16 +14,19 @@ import mcl.utils.cachix : cachixNixStoreUrl, DeploySpec, createMachineDeploySpec
 import mcl.utils.tui : bold;
 import mcl.utils.json : toJSON, fromJSON, tryDeserializeFromJsonFile;
 
-import mcl.commands.ci_matrix : flakeAttr, params, Params, nixEvalJobs, SupportedSystem;
+import mcl.commands.ci_matrix : flakeAttr, Params, nixEvalJobs, SupportedSystem;
 
 export void deploy_spec()
 {
+
     const deploySpecFile = resultDir.buildPath("cachix-deploy-spec.json");
 
     if (!exists(deploySpecFile))
     {
+        Params params = parseEnv!Params;
+        
         auto nixosConfigs = flakeAttr("legacyPackages", SupportedSystem.x86_64_linux, "bareMetalMachines")
-            .nixEvalJobs(params.cachixCache.cachixNixStoreUrl);
+            .nixEvalJobs(params, params.cachixCache.cachixNixStoreUrl);
 
         auto configsMissingFromCachix = nixosConfigs.filter!(c => !c.isCached);