chore(mcl): rework compare_disko command

Author: georgiyord

packages/mcl/src/src/mcl/commands/compare_disko.d

@@ -1,9 +1,13 @@
 module mcl.commands.compare_disko;
 
+import core.cpuid : threadsPerCPU;
+
 import mcl.utils.env : optional, parseEnv;
 import mcl.utils.nix : nix;
 import mcl.utils.path : getTopLevel;
+import mcl.utils.path : rootDir, resultDir, gcRootsDir, createResultDirs;
 import mcl.utils.process : execute;
+import mcl.utils.log : errorAndExit;
 
 import std.typecons : Tuple, tuple;
 import std.file : mkdirRecurse, exists, rmdirRecurse;
@@ -13,11 +17,14 @@ import std.json : JSONValue, parseJSON, JSONOptions;
 import std.file : write;
 import std.logger : tracef, errorf, infof;
 import std.process : environment;
-import mcl.utils.log : errorAndExit;
+import std.algorithm : map, filter, reduce, chunkBy, find, any, sort, startsWith, each, canFind, fold, uniq;
+import std.array;
+import std.meta : Filter;
 
 struct Params
 {
     @optional() string baseBranch;
+    @optional() int maxWorkers;
 
     void setup(){
         if (baseBranch == null){
@@ -26,27 +33,68 @@ struct Params
     }
 }
 
+enum ChangeStatus{
+    Unchanged,
+    Changed,
+    Removed,
+    New
+}
+
 struct MachineChanges{
     string machine;
-    bool _config;
-    bool _create;
+    ChangeStatus _config;
+    ChangeStatus _create;
+}
+
+template FilterFields(S) {
+    template isNotExcluded(string memberName) {
+        enum isNotExcluded = (memberName != "machine");
+    }
+    alias AllMembers = __traits(allMembers, S);
+    alias FilteredMembers = Filter!(isNotExcluded, AllMembers);
+    enum FieldNamesExcept = [ FilteredMembers ];
+}
+
+enum string[] diskoOptions = FilterFields!(MachineChanges).FieldNamesExcept;
+
+string statusToSymbol(ChangeStatus s) {
+    final switch(s){
+        case ChangeStatus.Unchanged:
+            return "🟩";
+            break;
+        case ChangeStatus.Changed:
+            return "⚡";
+            break;
+        case ChangeStatus.Removed:
+            return "🗑";
+            break;
+        case ChangeStatus.New:
+            return "🧩";
+            break;
+    }
 }
 
 // TODO: handle case where a machine is missing from one branch
 export void compare_disko(string[] args){
     nix.eval!JSONValue("", [], "");
     const params = parseEnv!Params;
+
     JSONValue configurations = nix.flake!JSONValue("", [], "show");
-    string[] machines;
+
+    string[] machinesNew;
     foreach (string k, JSONValue v; configurations["nixosConfigurations"]){
         if (k[$-3 .. $] != "-vm"            &&
             k != "gitlab-runner-container"  &&
             k != "minimal-container"        )
         {
-            machines ~= k;
+            machinesNew ~= k;
         }
     }
 
+    auto attr = constructCommandAttr("./.", machinesNew);
+    auto machineOptionsRootNew = nix.eval!JSONValue("", ["--impure", "--expr", attr]);
+
+
     string gitRoot = getTopLevel();
     string worktreeBaseBranch = gitRoot~"-"~params.baseBranch;
 
@@ -55,56 +103,61 @@ export void compare_disko(string[] args){
     }
 
     execute(["git", "worktree" , "add", worktreeBaseBranch, params.baseBranch]);
-    string freshTestsDir = gitRoot~"/disko-tests";
-    string staleTestsDir = worktreeBaseBranch~"/disko-tests";
-    mkdirRecurse(staleTestsDir);
-    mkdirRecurse(freshTestsDir);
 
-    Tuple!(string , string )[] machineDiffs;
+    configurations = nix.flake!JSONValue(worktreeBaseBranch, [], "show");
+
+    string[] machinesOld;
+    foreach (string k, JSONValue v; configurations["nixosConfigurations"]){
+        if (k[$-3 .. $] != "-vm"            &&
+            k != "gitlab-runner-container"  &&
+            k != "minimal-container"        )
+        {
+            machinesOld ~= k;
+        }
+    }
+
+    attr = constructCommandAttr(worktreeBaseBranch, machinesOld);
+    auto machineOptionsRootOld = nix.eval!JSONValue("", ["--impure", "--expr", attr]);
+
     MachineChanges[] machineChanges;
 
+    string[] machines = uniq(machinesOld ~ machinesNew).array;
+
     foreach (string m; machines){
         MachineChanges mc;
         mc.machine = m;
-        foreach (setting; __traits(allMembers, MachineChanges)){
-            static if (is(typeof(__traits(getMember, mc, setting)) == bool)){
-                string new_setting =
-                    nix.eval(gitRoot~"#nixosConfigurations."~m~".config.disko.devices."~setting, [
-                    "--option", "warn-dirty", "false",
-                    "--accept-flake-config"]);
-                tracef("CREATING %s_%s_new FILE", m, setting);
-                write(freshTestsDir~"/"~m~"_"~setting~"_new", new_setting);
-                string old_setting =
-                    nix.eval(worktreeBaseBranch~"#nixosConfigurations."~m~".config.disko.devices."~setting, [
-                    "--option", "warn-dirty", "false",
-                    "--accept-flake-config"]);
-                tracef("CREATING %s_%s_old FILE", m, setting);
-                write(staleTestsDir~"/"~m~"_"~setting~"_old", old_setting);
-
-
-                string diff = execute([
-                    "git", "--no-pager", "diff", "--no-index",
-                    staleTestsDir~"/"~m~"_"~setting~"_old",
-                    freshTestsDir~"/"~m~"_"~setting~"_new"]);
-
-                if (diff == ""){
+        if (m in machineOptionsRootOld.object && m in machineOptionsRootNew.object) {
+            static foreach (setting; diskoOptions){
+                if (machineOptionsRootOld[m][setting] == machineOptionsRootNew[m][setting]){
                     infof("✔ NO DIFFERENCE IN %s", m);
-                    __traits(getMember, mc, setting) = true;
+                    __traits(getMember, mc, setting) = ChangeStatus.Unchanged;
                 }
                 else{
                     infof("✖ DIFFERENCE IN %s", m);
-                    __traits(getMember, mc, setting) = false;
+                    __traits(getMember, mc, setting) = ChangeStatus.Changed;
                 }
             }
         }
+        else if (m in machineOptionsRootOld) {
+            static foreach (setting; diskoOptions){
+                infof("✖ MACHINE %s NO LONGER EXISTS", m);
+                __traits(getMember, mc, setting) = ChangeStatus.Removed;
+            }
+        }
+        else {
+            static foreach (setting; diskoOptions){
+                infof("✖ MACHINE %s IS NEW", m);
+                __traits(getMember, mc, setting) = ChangeStatus.New;
+            }
+        }
         machineChanges ~= mc;
     }
     infof("------------------------------------------------------");
-    if(machineDiffs.length == 0){
+    if(machineChanges.length == 0){
         infof("✔✔✔ NO CONFIGS WITH DIFFS");
     }
     else{
-        infof("✖✖✖ LIST OF CONFIGS WITH DIFFS");
+        infof("✖✖✖ LIST OF CONFIGS WITH DIFFS"); //TODO: HANDLE DIFFERENCES ON OPTION LEVEL WITH DIFFERENTIATION OF STATUS
         foreach(mc; machineChanges){
             infof(mc.machine);
         }
@@ -113,7 +166,6 @@ export void compare_disko(string[] args){
 
     // Cleanup
     execute(["git", "worktree" , "remove", worktreeBaseBranch, "--force"]);
-    rmdirRecurse(freshTestsDir);
 }
 
 void create_comment(MachineChanges[] machineChanges){
@@ -124,6 +176,11 @@ void create_comment(MachineChanges[] machineChanges){
     else{
         // TODO: Change the generation of the table to have as many collumns as fields in MachineChanges at compile time
         data ~= "\n\nBellow you will find a summary of machines and whether their disko attributes have differences.";
+        data ~= "\n\n**Legend:**";
+        data ~= "\n🟩 = No changes";
+        data ~= "\n⚡ = Something is different";
+        data ~= "\n🗑 = Has been removed";
+        data ~= "\n🧩 = Has been added";
 
         data ~= "\n\n";
         foreach (string field; __traits(allMembers, MachineChanges)){
@@ -137,18 +194,29 @@ void create_comment(MachineChanges[] machineChanges){
 
         foreach(mc; machineChanges){
             foreach (string field; __traits(allMembers, MachineChanges)){
-                static if (is(typeof(__traits(getMember, mc, field)) == bool)){
-                    data~="| " ~ (__traits(getMember, mc, field) ? "🟩" : "⚠️") ~ " ";
-                }
-                else static if (is(typeof(__traits(getMember, mc, field)) == string)){
+                static if (is(typeof(__traits(getMember, mc, field)) == string)){
                     data~="| " ~ __traits(getMember, mc, field) ~ " ";
                 }
-                else{
-                    assert(0);
+                else {
+                    data~="| " ~ statusToSymbol(__traits(getMember, mc, field)) ~ " ";
                 }
             }
             data ~= "|\n";
         }
     }
     write("comment.md", data);
 }
+
+
+string constructCommandAttr(string flakePath, string[] machines){
+    string attr = "let flake = (builtins.getFlake (builtins.toString " ~ flakePath ~ ")); in { ";
+    foreach (m; machines){
+        attr ~= m ~ " = { ";
+        foreach (option; diskoOptions){
+            attr ~= option ~ " = flake.nixosConfigurations." ~ m ~ ".config.disko.devices." ~ option ~ "; ";
+        }
+        attr ~= "}; ";
+    }
+    attr ~= "}";
+    return attr;
+}

packages/mcl/src/src/mcl/utils/nix.d

@@ -80,7 +80,7 @@ struct NixCommand
                 args = ["--json"] ~ args;
 
             auto command = [
-                "nix", "--experimental-features", "nix-command flakes",
+                "nix", "--experimental-features", "nix-command flakes pipe-operators",
                 commandName
             ] ~ (subcommand == "" ? [] : [ subcommand ]) ~ args ~ path;