testing more rigorous by having selectable sets of tags

Author: scivision

.github/workflows/ci.yml

@@ -37,7 +37,7 @@ jobs:
         - os: windows-latest
           release: R2025a
         - os: windows-latest
-          release: R2023a
+          release: R2023b
 
 
     steps:

.github/workflows/composite-mex/action.yml

@@ -7,8 +7,8 @@ runs:
   steps:
 
   - name: Mex Test
-    if: ${{ matrix.release >= 'R2023a' || startsWith(matrix.release, 'latest') }}
+    if: ${{ matrix.release >= 'R2023b' || startsWith(matrix.release, 'latest') }}
     uses: matlab-actions/run-build@v2
     with:
       startup-options: ${{ matrix.startup-options }} -logfile ${{ matrix.release }}-${{ runner.os }}-test.log
-      tasks: mex test
+      tasks: ${{ matrix.release < 'R2024b' && !startsWith(matrix.release, 'latest') && 'test_mex' || 'test:mex' }}

.github/workflows/composite-nomex/action.yml

@@ -7,22 +7,29 @@ runs:
   steps:
 
   - name: Non-Mex Test
-    if: ${{ matrix.release >= 'R2023a' || startsWith(matrix.release, 'latest') }}
+    if: ${{ matrix.release >= 'R2024b' || startsWith(matrix.release, 'latest') }}
     uses: matlab-actions/run-build@v2
     with:
       startup-options: ${{ matrix.startup-options }}
-      tasks: test
+      tasks: test:nomex ${{ !contains(matrix.startup-options, '-nojvm') && 'test:exe' }}
+
+
+  - name: Non-Mex Test (< R2024b)
+    if: ${{ matrix.release >= 'R2023b' && matrix.release < 'R2024b' && !startsWith(matrix.release, 'latest') }}
+    uses: matlab-actions/run-build@v2
+    with:
+      startup-options: ${{ matrix.startup-options }}
+      tasks: test_nomex ${{ !contains(matrix.startup-options, '-nojvm') && 'test_exe' }}
+
 
 # note: "source-folder" is necessary, but watch out as it adds all subfolders to path too
 # https://github.com/matlab-actions/run-tests?tab=readme-ov-file#run-matlab-tests
-  - name: Run tests (manual)
-    if: ${{ matrix.release < 'R2023a' && !startsWith(matrix.release, 'latest') }}
-    uses: matlab-actions/run-tests@v2
+  - name: Non-mex tests (< R2023b)
+    if: ${{ matrix.release < 'R2023b' && !startsWith(matrix.release, 'latest') }}
+    uses: matlab-actions/run-command@v2
     with:
-      source-folder: ${{ github.workspace }}
-      select-by-folder: test
-      strict: false
-      startup-options: -logfile ${{ matrix.release }}-${{ runner.os }}-test.log
+      command: test_nomex
+      startup-options: -sd ./test -logfile ${{ matrix.release }}-${{ runner.os }}-test.log
 
 
   - name: upload logfile

.gitignore

@@ -1,7 +1,6 @@
 Filesystem.class
 CodeIssues.sarif
-TestResults.xml
-code-coverage.xml
+*.xml
 
 resources/
 .buildtool/

buildfile.m

@@ -1,5 +1,6 @@
 function plan = buildfile
-assert(~isMATLABReleaseOlderThan("R2023a"), "MATLAB R2023a or newer is required for this buildfile")
+import matlab.unittest.selectors.HasTag
+assert(~isMATLABReleaseOlderThan("R2023b"), "MATLAB R2023b or newer is required for this buildfile")
 
 plan = buildplan(localfunctions);
 
@@ -8,14 +9,37 @@
 addpath(plan.RootFolder)
 
 %% Self-test setup
-if isMATLABReleaseOlderThan("R2023b")
-  plan("test") = matlab.buildtool.Task(Actions=@legacy_test);
-elseif isMATLABReleaseOlderThan("R2024a")
-  plan("test") = matlab.buildtool.tasks.TestTask("test", Strict=false);
+plan("clean") = matlab.buildtool.tasks.CleanTask;
+
+cnomex = ~HasTag("exe") & ~HasTag("mex");
+cmex = ~HasTag("exe") & HasTag("mex");
+
+if isMATLABReleaseOlderThan("R2024b")
+
+  plan("test_exe") = matlab.buildtool.tasks.TestTask("test", Tag="exe", Dependencies="exe");
+  plan("test_nomex") = matlab.buildtool.Task(Actions=@(context) legacy_test(context, cnomex), Dependencies="clean");
+  plan("test_mex") = matlab.buildtool.Task(Actions=@(context) legacy_test(context, cmex), Dependencies="mex");
+
+elseif isMATLABReleaseOlderThan("R2025a")
+
+  plan("test:exe")   = matlab.buildtool.tasks.TestTask("test", Tag="exe", Dependencies="exe");
+  plan("test:nomex") = matlab.buildtool.Task(Actions=@(context) legacy_test(context, cnomex), Dependencies="clean");
+  plan("test:mex")   = matlab.buildtool.Task(Actions=@(context) legacy_test(context, cmex), Dependencies="mex");
+
 else
   % can't use SourceFiles= if "mex" Task was run, even if plan("test").DisableIncremental = true;
   % this means incremental tests can't be used with MEX files (as of R2024b)
-  plan("test") = matlab.buildtool.tasks.TestTask("test", Strict=false, TestResults="TestResults.xml");
+  plan("test:exe")   = matlab.buildtool.tasks.TestTask("test", Tag="exe", ...
+                         Dependencies="exe", TestResults="TestResults_exe.xml", Strict=false);
+
+  plan("test:nomex") = matlab.buildtool.tasks.TestTask("test", ...
+                         Selector=cnomex, ...
+                         Dependencies="clean", TestResults="TestResults_nomex.xml", Strict=false);
+
+  plan("test:mex")   = matlab.buildtool.tasks.TestTask("test", ...
+                         Selector=cmex, ...
+                         Dependencies="mex", TestResults="TestResults_mex.xml", Strict=false);
+
 end
 
 td = plan.RootFolder + "/test";
@@ -26,11 +50,6 @@
   exes = [exes, td+"/stdout_stderr_fortran.exe", td+"/stdin_fortran.exe"];
 end
 plan("exe") = matlab.buildtool.Task(Inputs=srcs, Outputs=exes, Actions=@build_exe);
-plan("test").Dependencies = "exe";
-
-if ~isMATLABReleaseOlderThan("R2023b")
-  plan("clean") = matlab.buildtool.tasks.CleanTask;
-end
 
 if ~isMATLABReleaseOlderThan("R2024a")
   plan("check") = matlab.buildtool.tasks.CodeIssuesTask(pkg_name, IncludeSubfolders=true, ...
@@ -82,9 +101,15 @@ function legacy_mex(context, compiler_opt, linker_opt)
 end
 
 
-function legacy_test(context)
-r = runtests(context.Plan.RootFolder + "/test", Strict=false);
-% Parallel Computing Toolbox takes more time to startup than is worth it for this task
+function legacy_test(context, sel)
+import matlab.unittest.TestSuite
+
+suite = TestSuite.fromFolder(context.Plan.RootFolder + "/test");
+suite = suite.selectIf(sel);
+
+runner = testrunner();
+
+r = run(runner, suite);
 
 assert(~isempty(r), "No tests were run")
 assertSuccess(r)
@@ -256,10 +281,6 @@ function build_exe(context)
 if msvc
   std = "/std:c++17";
   % on Windows, Matlab doesn't register unsupported MSVC or oneAPI
-elseif cxx.Name == "Xcode Clang++"
-  if isMATLABReleaseOlderThan("R2023b") && stdlib.version_atleast(cxx.Version, "15.0")
-    warning("Xcode Clang++ " + cxx.Version + " may not support this Matlab version")
-  end
 elseif ~strlength(compiler_id) && cxx.ShortName == "g++"
   if ~stdlib.version_atleast(cxx.Version, "8")
     warning("g++ 8 or newer is required for MEX, detected g++" + cxx.Version)

test/TestDisk.m

@@ -4,10 +4,23 @@
 Ps = {".", "", "not-exist"}
 end
 
+methods(Test, TestTags = "mex")
+
+function test_mex_disk_available(tc)
+import matlab.unittest.constraints.IsFile
+tc.assertThat(fileparts(mfilename("fullpath")) + "/../+stdlib/disk_available." + mexext, IsFile)
+end
+
+function test_mex_disk_capacity(tc)
+import matlab.unittest.constraints.IsFile
+tc.assertThat(fileparts(mfilename("fullpath")) + "/../+stdlib/disk_capacity." + mexext, IsFile)
+end
+
+end
+
 methods (Test)
 
 function test_disk_available(tc, Ps)
-tc.assumeTrue(isfile(fileparts(mfilename("fullpath")) + "/../+stdlib/disk_available." + mexext))
 
 zero = uint64(0);
 
@@ -19,7 +32,6 @@ function test_disk_available(tc, Ps)
 end
 
 function test_disk_capacity(tc, Ps)
-tc.assumeTrue(isfile(fileparts(mfilename("fullpath")) + "/../+stdlib/disk_capacity." + mexext))
 
 zero = uint64(0);
 

test/TestMex.m

@@ -1,11 +1,8 @@
 classdef TestMex < matlab.unittest.TestCase
 
-methods (Test)
+methods (Test, TestTags = "mex")
 
 function test_is_char_device(tc)
-
-tc.assumeTrue(isfile(fileparts(mfilename("fullpath")) + "/../+stdlib/is_char_device." + mexext))
-
 % /dev/stdin may not be available on CI systems
 if ispc
   n = "NUL";
@@ -18,14 +15,13 @@ function test_is_char_device(tc)
 
 
 function test_is_admin(tc)
-tc.assumeTrue(isfile(fileparts(mfilename("fullpath")) + "/../+stdlib/is_admin." + mexext))
-
 tc.verifyClass(stdlib.is_admin(), "logical")
 end
 
 
 function test_remove_file(tc)
-tc.assumeTrue(isfile(fileparts(mfilename("fullpath")) + "/../+stdlib/remove." + mexext))
+import matlab.unittest.constraints.IsFile
+tc.assertThat(fileparts(mfilename("fullpath")) + "/../+stdlib/remove." + mexext, IsFile)
 
 d = tc.createTemporaryFolder();
 
@@ -39,7 +35,8 @@ function test_remove_file(tc)
 
 
 function test_remove_empty_dir(tc)
-tc.assumeTrue(isfile(fileparts(mfilename("fullpath")) + "/../+stdlib/remove." + mexext))
+import matlab.unittest.constraints.IsFile
+tc.assertThat(fileparts(mfilename("fullpath")) + "/../+stdlib/remove." + mexext, IsFile)
 
 d = tc.createTemporaryFolder();
 
@@ -48,7 +45,8 @@ function test_remove_empty_dir(tc)
 
 
 function test_remove_recursive(tc)
-tc.assumeTrue(isfile(fileparts(mfilename("fullpath")) + "/../+stdlib/remove." + mexext))
+import matlab.unittest.constraints.IsFile
+tc.assertThat(fileparts(mfilename("fullpath")) + "/../+stdlib/remove." + mexext, IsFile)
 
 d = tc.createTemporaryFolder();
 

test/TestPermissions.m

@@ -4,6 +4,17 @@
 Ps = {".", tempname(), "", "not-exist"}
 end
 
+methods(Test, TestTags = "mex")
+
+function test_mex_set_permissions(tc)
+import matlab.unittest.constraints.IsFile
+tc.assertThat(fileparts(mfilename("fullpath")) + "/../+stdlib/set_permissions." + mexext, IsFile)
+end
+
+
+end
+
+
 methods (Test)
 
 function test_get_permissions(tc, Ps)
@@ -26,8 +37,6 @@ function test_set_permissions(tc)
 
 import matlab.unittest.constraints.StartsWithSubstring
 
-tc.assumeTrue(isfile(fileparts(mfilename("fullpath")) + "/../+stdlib/set_permissions." + mexext))
-
 tf = tc.createTemporaryFolder();
 
 nr = fullfile(tf, "no-read");

test/TestRelative.m

@@ -5,22 +5,20 @@
 pp = init_prox()
 end
 
-methods(TestClassSetup)
-function mex_required(tc)
-tc.assumeTrue(isfile(fileparts(mfilename("fullpath")) + "/../+stdlib/relative_to." + mexext))
-tc.assumeTrue(isfile(fileparts(mfilename("fullpath")) + "/../+stdlib/proximate_to." + mexext))
-end
-end
-
-
-methods (Test)
+methods (Test, TestTags = "mex")
 
 function test_relative_to(tc, pr)
+import matlab.unittest.constraints.IsFile
+tc.assertThat(fileparts(mfilename("fullpath")) + "/../+stdlib/relative_to." + mexext, IsFile)
+
 tc.verifyEqual(stdlib.relative_to(pr{1}, pr{2}), pr{3}, ...
   "relative_to(" + pr{1} + "," + pr{2}+")")
 end
 
 function test_proximate_to(tc, pp)
+import matlab.unittest.constraints.IsFile
+tc.assertThat(fileparts(mfilename("fullpath")) + "/../+stdlib/proximate_to." + mexext, IsFile)
+
 tc.verifyEqual(stdlib.proximate_to(pp{1}, pp{2}), pp{3}, ...
   "proximate_to(" + pp{1} + "," + pp{2}+")")
 end

test/TestSubprocess.m

@@ -12,14 +12,14 @@ function java_required(tc)
 end
 
 
-methods (Test)
+methods (Test, TestTags="exe")
 
 function test_stdout_stderr(tc, lang_out)
 import matlab.unittest.constraints.IsFile
 
 cwd = fileparts(mfilename('fullpath'));
 exe = cwd + "/stdout_stderr_" + lang_out + ".exe";
-tc.assumeThat(exe, IsFile, exe + " not found")
+tc.assumeThat(exe, IsFile)
 
 [status, msg, err] = stdlib.subprocess_run(exe);
 tc.assertEqual(status, 0, err)
@@ -29,11 +29,10 @@ function test_stdout_stderr(tc, lang_out)
 
 
 function test_stdin(tc, lang_in)
-import matlab.unittest.constraints.IsFile
 
 cwd = fileparts(mfilename('fullpath'));
 exe = cwd + "/stdin_" + lang_in + ".exe";
-tc.assumeThat(exe, IsFile, exe + " not found")
+tc.assumeThat(exe, matlab.unittest.constraints.IsFile)
 
 [status, msg, err] = stdlib.subprocess_run(exe, stdin="1 2");
 
@@ -67,11 +66,10 @@ function test_cwd(tc)
 
 
 function test_env_run(tc)
-import matlab.unittest.constraints.IsFile
 
 cwd = fileparts(mfilename('fullpath'));
 exe = cwd + "/printenv.exe";
-tc.assumeThat(exe, IsFile, exe + " not found")
+tc.assumeThat(exe, matlab.unittest.constraints.IsFile)
 
 names = ["TEST1", "TEST2"];
 vals = ["test123", "test321"];
@@ -89,11 +87,10 @@ function test_env_run(tc)
 
 function test_timeout(tc)
 import matlab.unittest.constraints.StartsWithSubstring
-import matlab.unittest.constraints.IsFile
 
 cwd = fileparts(mfilename('fullpath'));
 exe = cwd + "/sleep.exe";
-tc.assumeThat(exe, IsFile, exe + " not found")
+tc.assumeThat(exe, matlab.unittest.constraints.IsFile)
 
 [ret, ~, err] = stdlib.subprocess_run(exe, timeout=1, stdout=false, stderr=false);