Additional commands. Fix model_dir bug when single folder name

README.md

@@ -68,6 +68,18 @@ The following commands are now available:
   Note that the generated HTML file is self-contained.
 
 
+* `compare_gout_files` -> list all gout paths that differ between two folders.
+  ```
+  compare_gout_files -h
+  ```
+  Does not require a ```--gout``` arg to ```runmodels```.
+
+
+* `show_diff_gout` -> like `compare_gout_files` but also does diffgout for each.
+  ```
+  show_diff_gout -h
+  ```
+
 * `modeldb-config` -> list configuration for `nrn-modeldb-ci`:
   ```
     modeldb-config

modeldb/commands.py

@@ -16,6 +16,55 @@
 from .modelrun import is_dir_non_empty
 from .modelrun import ModelRunManager
 from .report import diff_reports
+import json
+import shutil
+import filecmp
+
+
+def _compare_gout_files_internal(
+    folder1: str, folder2: str
+) -> tuple[list[str], list[str], list[str]]:
+    """
+    Internal helper to compare 'gout' files between two folders.
+
+    Args:
+        folder1 (str): Path to first folder
+        folder2 (str): Path to second folder
+
+    Returns:
+        tuple: (files_only_in_folder1, files_only_in_folder2, files_different_content)
+    """
+    # Convert to Path objects
+    folder1_path = Path(folder1)
+    folder2_path = Path(folder2)
+
+    # Find all gout files in both folders
+    gout_files1 = {
+        str(p.relative_to(folder1_path))
+        for p in folder1_path.rglob("gout")
+        if p.is_file()
+    }
+    gout_files2 = {
+        str(p.relative_to(folder2_path))
+        for p in folder2_path.rglob("gout")
+        if p.is_file()
+    }
+
+    # Find files unique to each folder
+    only_in_folder1 = list(gout_files1 - gout_files2)
+    only_in_folder2 = list(gout_files2 - gout_files1)
+
+    # Find common files that differ in content
+    common_files = gout_files1 & gout_files2
+    different_content = []
+
+    for file_path in common_files:
+        file1 = folder1_path / file_path
+        file2 = folder2_path / file_path
+        if not filecmp.cmp(file1, file2, shallow=False):
+            different_content.append(file_path)
+
+    return sorted(only_in_folder1), sorted(only_in_folder2), sorted(different_content)
 
 
 def runmodels(args=None):
@@ -327,3 +376,112 @@ def diffreports2html(args=None):
         == 1
     )
     return code
+
+
+def compare_gout_files(args=None):
+    """compare_gout_files
+
+    Compare 'gout' files between two folders and list files that differ.
+
+    Usage:
+        compare_gout_files <folder1> <folder2>
+        compare_gout_files -h         Print help
+
+    Arguments:
+        folder1=PATH      Required: path to first folder containing gout files
+        folder2=PATH      Required: path to second folder containing gout files
+
+    Examples:
+        compare_gout_files 3246-master 3246-8.0.2
+    """
+    options = docopt(compare_gout_files.__doc__, args)
+
+    folder1 = options.pop("<folder1>")
+    folder2 = options.pop("<folder2>")
+
+    only_in_folder1, only_in_folder2, different_content = _compare_gout_files_internal(
+        folder1, folder2
+    )
+
+    # Print results
+    print("Files only in {}:".format(folder1))
+    for f in only_in_folder1:
+        print(f"  {f}")
+
+    print("\nFiles only in {}:".format(folder2))
+    for f in only_in_folder2:
+        print(f"  {f}")
+
+    print("\nFiles in both folders with different content:")
+    for f in different_content:
+        print(f"  {f}")
+
+    # Return status code
+    return 1 if only_in_folder1 or only_in_folder2 or different_content else 0
+
+
+def show_diff_gout(args=None):
+    """show_diff_gout
+
+    Compare 'gout' files between two folders, list differences, and show graphical comparisons one at a time.
+
+    Usage:
+        show_diff_gout <folder1> <folder2>
+        show_diff_gout -h         Print help
+
+    Arguments:
+        folder1=PATH      Required: path to first folder containing gout files
+        folder2=PATH      Required: path to second folder containing gout files
+
+    Examples:
+        show_diff_gout 8.2.6 827
+    """
+    options = docopt(show_diff_gout.__doc__, args)
+
+    folder1 = options.pop("<folder1>")
+    folder2 = options.pop("<folder2>")
+
+    # Get the list of differing files
+    only_in_folder1, only_in_folder2, different_content = _compare_gout_files_internal(
+        folder1, folder2
+    )
+
+    # Print results (same as compare_gout_files)
+    print("Files only in {}:".format(folder1))
+    for f in only_in_folder1:
+        print(f"  {f}")
+
+    print("\nFiles only in {}:".format(folder2))
+    for f in only_in_folder2:
+        print(f"  {f}")
+
+    print("\nFiles in both folders with different content:")
+    for f in different_content:
+        print(f"  {f}")
+
+    # Show graphical comparisons for files with different content
+    if different_content:
+        print(
+            "\nShowing graphical comparisons (press Enter after closing each NEURON window to continue)..."
+        )
+        folder1_path = Path(folder1)
+        folder2_path = Path(folder2)
+
+        for file_path in different_content:
+            gout_file1 = str(folder1_path / file_path)
+            gout_file2 = str(folder2_path / file_path)
+
+            print(f"\nComparing: {file_path}")
+            cmd = 'nrngui -c "strdef gout1" -c "gout1=\\"{}\\"" -c "strdef gout2" -c "gout2=\\"{}\\"" modeldb/showgout.hoc'.format(
+                gout_file1, gout_file2
+            )
+            commands = shlex.split(cmd)
+            # Run and wait for the process to complete (user closes NEURON GUI)
+            process = subprocess.Popen(commands)
+            process.wait()
+
+            # Prompt for user input to continue
+            input("Press Enter to continue to the next comparison...")
+
+    # Return status code
+    return 1 if only_in_folder1 or only_in_folder2 or different_content else 0

modeldb/modeldb-run.yaml

@@ -1460,8 +1460,10 @@
     - "model;optmz"
   skip: true
 187473:
-  model_dir:
-    - mods
+    model_dir: mods
+    script:
+    # prevent hanging because of matplotlib window
+    - sed -i 's,^system,//system,' mosinit.hoc
 187604:
   curate_patterns:
     - pattern: 'TIME HOST 0: [0-9\.]+ seconds \(set up\)'
@@ -2039,6 +2041,23 @@
 2016662:
   skip: true
   comment: "Not on GitHub and dirs contain spaces"
+2016664:
+  model_dir: _mod
+2017402:
+  skip: true
+  comment: "Error: new_seed used as both variable and function in file Gfluc.mod"
+2018247:
+  skip: true
+  comment: "Not on GitHub and dirs contain spaces"
+2018260:
+  skip: true
+  comment: "Jupyter notebook. Requires conda"
+2018263:
+  skip: true
+  comment: "See README.md"
+2019342:
+  skip: true
+  comment: "branching.mod needs updating"
 139150:
   github: 'pull/1'
 185513:

modeldb/modelrun.py

@@ -320,7 +320,13 @@ def run_model(model):
 
         if "model_dir" in model:
             # go over all of the `model_dir`s and group the mod files together for compilation
-            for model_dir in model["model_dir"]:
+            # Handle model["model_dir"] as a string or list
+            model_dirs = (
+                model["model_dir"].split(";")
+                if isinstance(model["model_dir"], str)
+                else model["model_dir"]
+            )
+            for model_dir in model_dirs:
                 mod_groups.append(
                     find_modfile_group(
                         Path(model.model_dir) / item for item in model_dir.split(";")

modeldb/report.py

@@ -25,8 +25,8 @@ def curate_run_data(run_data, model=None):
         # nrniv: unable to open font "*helvetica-medium-r-normal*--14*", using "fixed" <-> special: unableto open font "*helvetica-medium-r-normal*--14*", using "fixed"
         "^nrniv:": "%neuron-executable%:",
         "^special:": "%neuron-executable%:",
-        "(Mon|Tue|Wed|Thu|Fri|Sat|Sun) (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+\d+\s+\d+:\d+:\d+ [A-Z\s]+ \d+": "%date_command%",
-        "total run time [0-9\.]+": "total run time %run_time%",
+        r"(Mon|Tue|Wed|Thu|Fri|Sat|Sun) (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+\d+\s+\d+:\d+:\d+ [A-Z\s]+ \d+": "%date_command%",
+        r"total run time [0-9\.]+": "total run time %run_time%",
         "(^.*distutils.*$)": "",
         "/.*?/lib/python.*/site-packages/": "%python-site-packages%",
     }

perfcmp1.py

@@ -0,0 +1,66 @@
+# python -i perfcmp1.py 8.2 master
+# Plots linear and log-log nrniv runtimes.
+
+import sys
+
+xaxis = sys.argv[-2]
+yaxis = sys.argv[-1]
+
+
+def rd(logfile):
+    f = open(logfile, "r")
+    lines = f.readlines()
+    data = {}
+    for line in lines:
+        if "Done for:" in line:
+            x = line.split()
+            try:
+                x = [x[i] for i in [7, 10, 12]]
+                x = [x[0], float(x[1][:-1]), float(x[2][:-1])]
+                data[x[0]] = x[1:]
+            except:
+                pass
+    return data
+
+
+j8 = rd(xaxis + ".log")
+master = rd(yaxis + ".log")
+
+combine = []
+for id in j8:
+    if id in master:
+        combine.append([id, j8[id][1], master[id][1]])
+
+combine.sort(key=lambda x: x[1])
+
+
+def f(x, logar):
+    return h.log10(x) if logar else x
+
+
+from neuron import h, gui
+
+
+def grph(logar):
+    g = h.Graph()
+    g.beginline(2, 1)
+    for x in [0.001, 0.01, 0.1, 1.0, 10.0, 100.0]:
+        g.line(f(x, logar), f(x, logar))
+    g.flush()
+    for i in combine:
+        g.mark(f(i[1], logar), f(i[2], logar), "O", 6, 1, 1)
+    g.exec_menu("View = plot")
+    xlab = "log10(%s time)" % (xaxis,) if logar else "%s time(s)" % (xaxis,)
+    ylab = "log10(%s time)" % (yaxis,) if logar else "%s time(s)" % (yaxis,)
+    g.label(f(0.001, logar), f(100.0, logar), ylab, 1, 1, 0, 0, 1)
+    g.label(f(100.0, logar), f(0.001, logar), xlab, 1, 1, 1, 0, 1)
+    return g
+
+
+print(f"Number of runmodel {len(j8.values())}")
+dat = [j8, master]
+for d, name in enumerate([xaxis, yaxis]):
+    for istyle, style in enumerate(["nrnivmodl", "nrniv"]):
+        print(f"total {style} for {name} : {sum([x[istyle] for x in dat[d].values()])}")
+
+z = [grph(0), grph(1)]

setup.py

@@ -44,6 +44,8 @@ def setup_package():
                 "modeldb-config = modeldb.commands:modeldb_config",
                 "report2html = modeldb.commands:report2html",
                 "diffreports2html = modeldb.commands:diffreports2html",
+                "compare_gout_files = modeldb.commands:compare_gout_files",
+                "show_diff_gout = modeldb.commands:show_diff_gout",
             ]
         ),
         long_description=long_description,

zperfcmp.py

@@ -0,0 +1,69 @@
+# python -i zperfcmp.py 8.2 master
+# Requires nrn branch hines/8.2-runtime and/or hines/master-runtime
+# Plots linear and log-log  accumulated fadvance, pc.psolve, cvode.solve
+# runtimes.
+
+import json
+import sys
+
+xaxis = sys.argv[-2]
+yaxis = sys.argv[-1]
+
+
+def rd(jsfile):
+    f = open(jsfile, "r")
+    js = json.load(f)
+    data = {}
+    for id in js:
+        if "nrn_run" in js[id]:
+            lines = js[id]["nrn_run"]
+            for line in lines:
+                if "ZZZZ" in line:
+                    x = line.split()
+                    try:
+                        if x[-2] == "runtime":
+                            data[id] = float(x[-1][:-1])  # eliminate trailing"
+                    except:
+                        pass
+    return data
+
+
+j8 = rd(xaxis + ".json")
+master = rd(yaxis + ".json")
+
+combine = []
+for id in j8:
+    if id in master:
+        combine.append([id, j8[id], master[id]])
+
+combine.sort(key=lambda x: x[1])
+
+
+def f(x, logar):
+    return h.log10(x) if logar else x
+
+
+from neuron import h, gui
+
+
+def grph(logar):
+    g = h.Graph()
+    g.beginline(2, 1)
+    for x in [0.001, 0.01, 0.1, 1.0, 10.0, 100.0]:
+        g.line(f(x, logar), f(x, logar))
+    g.flush()
+    for i in combine:
+        g.mark(f(i[1], logar), f(i[2], logar), "O", 6, 1, 1)
+    g.exec_menu("View = plot")
+    xlab = "log10(%s time)" % (xaxis,) if logar else "%s time(s)" % (xaxis,)
+    ylab = "log10(%s time)" % (yaxis,) if logar else "%s time(s)" % (yaxis,)
+    g.label(f(0.001, logar), f(100.0, logar), ylab, 1, 1, 0, 0, 1)
+    g.label(f(100.0, logar), f(0.001, logar), xlab, 1, 1, 1, 0, 1)
+    return g
+
+
+print(f"Number of runtimes {len(combine)}")
+for i, name in enumerate([xaxis, yaxis]):
+    print(f"total runtime for {name} : {sum([x[i + 1] for x in combine])}")
+
+z = [grph(0), grph(1)]