testing more rigorous by having selectable sets of tags

Author: scivision

.github/workflows/ci-nojvm.yml

@@ -7,6 +7,7 @@ on:
     - "**.h"
     - "**.cpp"
     - ".github/workflows/ci-nojvm.yml"
+    - ".github/workflows/*/action.yml"
     - "!private/publish_gen_index_html.m"
     - "!octave_build.m"
     - "!scripts/**"
@@ -41,6 +42,3 @@ jobs:
 
     - name: No Java, no Mex tests
       uses: ./.github/workflows/composite-nomex
-
-    - name: No Java, with MEX tests
-      uses: ./.github/workflows/composite-mex

.github/workflows/ci.yml

@@ -7,6 +7,7 @@ on:
     - "**.h"
     - "**.cpp"
     - ".github/workflows/ci.yml"
+    - ".github/workflows/*/action.yml"
     - "!private/publish_gen_index_html.m"
     - "!octave_build.m"
     - "!scripts/**"
@@ -37,7 +38,7 @@ jobs:
         - os: windows-latest
           release: R2025a
         - os: windows-latest
-          release: R2023a
+          release: R2023b
 
 
     steps:

.github/workflows/composite-mex/action.yml

@@ -7,8 +7,8 @@ runs:
   steps:
 
   - name: Mex Test
-    if: ${{ matrix.release >= 'R2023a' || startsWith(matrix.release, 'latest') }}
+    if: ${{ matrix.release >= 'R2023b' || startsWith(matrix.release, 'latest') }}
     uses: matlab-actions/run-build@v2
     with:
       startup-options: ${{ matrix.startup-options }} -logfile ${{ matrix.release }}-${{ runner.os }}-test.log
-      tasks: mex test
+      tasks: ${{ matrix.release < 'R2024b' && !startsWith(matrix.release, 'latest') && 'test_mex test_exe' || 'test:mex test:exe' }}

.github/workflows/composite-nomex/action.yml

@@ -7,22 +7,29 @@ runs:
   steps:
 
   - name: Non-Mex Test
-    if: ${{ matrix.release >= 'R2023a' || startsWith(matrix.release, 'latest') }}
+    if: ${{ matrix.release >= 'R2024b' || startsWith(matrix.release, 'latest') }}
     uses: matlab-actions/run-build@v2
     with:
       startup-options: ${{ matrix.startup-options }}
-      tasks: test
+      tasks: ${{ contains(matrix.startup-options, '-nojvm') && 'test:nojavamex' || 'test:nomex' }}
+
+
+  - name: Non-Mex Test (< R2024b)
+    if: ${{ matrix.release >= 'R2023b' && matrix.release < 'R2024b' && !startsWith(matrix.release, 'latest') }}
+    uses: matlab-actions/run-build@v2
+    with:
+      startup-options: ${{ matrix.startup-options }}
+      tasks: test_nomex
+
 
 # note: "source-folder" is necessary, but watch out as it adds all subfolders to path too
 # https://github.com/matlab-actions/run-tests?tab=readme-ov-file#run-matlab-tests
-  - name: Run tests (manual)
-    if: ${{ matrix.release < 'R2023a' && !startsWith(matrix.release, 'latest') }}
-    uses: matlab-actions/run-tests@v2
+  - name: Non-mex tests (< R2023b)
+    if: ${{ matrix.release < 'R2023b' && !startsWith(matrix.release, 'latest') }}
+    uses: matlab-actions/run-command@v2
     with:
-      source-folder: ${{ github.workspace }}
-      select-by-folder: test
-      strict: false
-      startup-options: -logfile ${{ matrix.release }}-${{ runner.os }}-test.log
+      command: addpath("${{ github.workspace }}"), test_nomex
+      startup-options: -sd ./test -logfile ${{ matrix.release }}-${{ runner.os }}-test.log
 
 
   - name: upload logfile

.gitignore

@@ -1,7 +1,6 @@
 Filesystem.class
 CodeIssues.sarif
-TestResults.xml
-code-coverage.xml
+*.xml
 
 resources/
 .buildtool/

buildfile.m

@@ -1,5 +1,6 @@
 function plan = buildfile
-assert(~isMATLABReleaseOlderThan("R2023a"), "MATLAB R2023a or newer is required for this buildfile")
+import matlab.unittest.selectors.HasTag
+assert(~isMATLABReleaseOlderThan("R2023b"), "MATLAB R2023b or newer is required for this buildfile")
 
 plan = buildplan(localfunctions);
 
@@ -8,14 +9,49 @@
 addpath(plan.RootFolder)
 
 %% Self-test setup
-if isMATLABReleaseOlderThan("R2023b")
-  plan("test") = matlab.buildtool.Task(Actions=@legacy_test);
-elseif isMATLABReleaseOlderThan("R2024a")
-  plan("test") = matlab.buildtool.tasks.TestTask("test", Strict=false);
+plan("clean") = matlab.buildtool.tasks.CleanTask;
+
+cnomex = ~HasTag("exe") & ~HasTag("mex");
+if isMATLABReleaseOlderThan("R2024b")
+  cnomex = cnomex & ~HasTag("symlink");
+end
+
+cmex = ~HasTag("exe") & HasTag("mex");
+cnojavamex = ~HasTag("java") & cnomex;
+
+
+if isMATLABReleaseOlderThan("R2024b")
+
+  plan("test_exe") = matlab.buildtool.tasks.TestTask("test", Tag="exe", Dependencies="exe");
+  plan("test_nomex") = matlab.buildtool.Task(Actions=@(context) legacy_test(context, cnomex), Dependencies="clean");
+  plan("test_mex") = matlab.buildtool.Task(Actions=@(context) legacy_test(context, cmex), Dependencies="mex");
+  plan("test_nojavamex") = matlab.buildtool.Task(Actions=@(context) legacy_test(context, cnojavamex), Dependencies="clean");
+
+elseif isMATLABReleaseOlderThan("R2025a")
+
+  plan("test:exe")   = matlab.buildtool.tasks.TestTask("test", Tag="exe", Dependencies="exe");
+  plan("test:nomex") = matlab.buildtool.Task(Actions=@(context) legacy_test(context, cnomex), Dependencies="clean");
+  plan("test:mex")   = matlab.buildtool.Task(Actions=@(context) legacy_test(context, cmex), Dependencies="mex");
+  plan("test:nojavamex") = matlab.buildtool.Task(Actions=@(context) legacy_test(context, cnojavamex), Dependencies="clean");
+
 else
   % can't use SourceFiles= if "mex" Task was run, even if plan("test").DisableIncremental = true;
   % this means incremental tests can't be used with MEX files (as of R2024b)
-  plan("test") = matlab.buildtool.tasks.TestTask("test", Strict=false, TestResults="TestResults.xml");
+  plan("test:exe")   = matlab.buildtool.tasks.TestTask("test", Tag="exe", ...
+                         Dependencies="exe", TestResults="TestResults_exe.xml", Strict=false);
+
+  plan("test:nomex") = matlab.buildtool.tasks.TestTask("test", ...
+                         Selector=cnomex, ...
+                         Dependencies="clean", TestResults="TestResults_nomex.xml", Strict=false);
+
+  plan("test:mex")   = matlab.buildtool.tasks.TestTask("test", ...
+                         Selector=cmex, ...
+                         Dependencies="mex", TestResults="TestResults_mex.xml", Strict=false);
+
+  plan("test:nojavamex") = matlab.buildtool.tasks.TestTask("test", ...
+                         Selector=cnojavamex, ...
+                         Dependencies="clean", TestResults="TestResults_nojavamex.xml", Strict=false);
+
 end
 
 td = plan.RootFolder + "/test";
@@ -26,11 +62,6 @@
   exes = [exes, td+"/stdout_stderr_fortran.exe", td+"/stdin_fortran.exe"];
 end
 plan("exe") = matlab.buildtool.Task(Inputs=srcs, Outputs=exes, Actions=@build_exe);
-plan("test").Dependencies = "exe";
-
-if ~isMATLABReleaseOlderThan("R2023b")
-  plan("clean") = matlab.buildtool.tasks.CleanTask;
-end
 
 if ~isMATLABReleaseOlderThan("R2024a")
   plan("check") = matlab.buildtool.tasks.CodeIssuesTask(pkg_name, IncludeSubfolders=true, ...
@@ -82,9 +113,15 @@ function legacy_mex(context, compiler_opt, linker_opt)
 end
 
 
-function legacy_test(context)
-r = runtests(context.Plan.RootFolder + "/test", Strict=false);
-% Parallel Computing Toolbox takes more time to startup than is worth it for this task
+function legacy_test(context, sel)
+import matlab.unittest.TestSuite
+
+suite = TestSuite.fromFolder(context.Plan.RootFolder + "/test");
+suite = suite.selectIf(sel);
+
+runner = testrunner();
+
+r = run(runner, suite);
 
 assert(~isempty(r), "No tests were run")
 assertSuccess(r)
@@ -256,10 +293,6 @@ function build_exe(context)
 if msvc
   std = "/std:c++17";
   % on Windows, Matlab doesn't register unsupported MSVC or oneAPI
-elseif cxx.Name == "Xcode Clang++"
-  if isMATLABReleaseOlderThan("R2023b") && stdlib.version_atleast(cxx.Version, "15.0")
-    warning("Xcode Clang++ " + cxx.Version + " may not support this Matlab version")
-  end
 elseif ~strlength(compiler_id) && cxx.ShortName == "g++"
   if ~stdlib.version_atleast(cxx.Version, "8")
     warning("g++ 8 or newer is required for MEX, detected g++" + cxx.Version)

test/TestDisk.m

@@ -4,10 +4,23 @@
 Ps = {".", "", "not-exist"}
 end
 
-methods (Test)
+methods(Test, TestTags = "mex")
+
+function test_mex_disk_available(tc)
+import matlab.unittest.constraints.IsFile
+tc.assertThat(fileparts(mfilename("fullpath")) + "/../+stdlib/disk_available." + mexext, IsFile)
+end
+
+function test_mex_disk_capacity(tc)
+import matlab.unittest.constraints.IsFile
+tc.assertThat(fileparts(mfilename("fullpath")) + "/../+stdlib/disk_capacity." + mexext, IsFile)
+end
+
+end
+
+methods (Test, TestTags = "java")
 
 function test_disk_available(tc, Ps)
-tc.assumeTrue(isfile(fileparts(mfilename("fullpath")) + "/../+stdlib/disk_available." + mexext))
 
 zero = uint64(0);
 
@@ -19,7 +32,6 @@ function test_disk_available(tc, Ps)
 end
 
 function test_disk_capacity(tc, Ps)
-tc.assumeTrue(isfile(fileparts(mfilename("fullpath")) + "/../+stdlib/disk_capacity." + mexext))
 
 zero = uint64(0);
 

test/TestHash.m

@@ -5,14 +5,8 @@
       {'sha-256', '2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824'}}
 end
 
-methods(TestClassSetup)
-function java_required(tc)
-tc.assumeTrue(stdlib.has_java())
-end
-end
-
 
-methods (Test)
+methods (Test, TestTags="java")
 
 
 function test_hash_text(tc, Ph)

test/TestJava.m

@@ -4,14 +4,8 @@
 Ps = {"."}
 end
 
-methods(TestClassSetup)
-function java_required(tc)
-tc.assumeTrue(stdlib.has_java())
-end
-end
-
 
-methods(Test)
+methods(Test, TestTags="java")
 
 function test_inode(tc)
 tc.assumeFalse(ispc(), "not for Windows")

test/TestMex.m

@@ -1,11 +1,8 @@
 classdef TestMex < matlab.unittest.TestCase
 
-methods (Test)
+methods (Test, TestTags = "mex")
 
 function test_is_char_device(tc)
-
-tc.assumeTrue(isfile(fileparts(mfilename("fullpath")) + "/../+stdlib/is_char_device." + mexext))
-
 % /dev/stdin may not be available on CI systems
 if ispc
   n = "NUL";
@@ -18,14 +15,13 @@ function test_is_char_device(tc)
 
 
 function test_is_admin(tc)
-tc.assumeTrue(isfile(fileparts(mfilename("fullpath")) + "/../+stdlib/is_admin." + mexext))
-
 tc.verifyClass(stdlib.is_admin(), "logical")
 end
 
 
 function test_remove_file(tc)
-tc.assumeTrue(isfile(fileparts(mfilename("fullpath")) + "/../+stdlib/remove." + mexext))
+import matlab.unittest.constraints.IsFile
+tc.assertThat(fileparts(mfilename("fullpath")) + "/../+stdlib/remove." + mexext, IsFile)
 
 d = tc.createTemporaryFolder();
 
@@ -39,7 +35,8 @@ function test_remove_file(tc)
 
 
 function test_remove_empty_dir(tc)
-tc.assumeTrue(isfile(fileparts(mfilename("fullpath")) + "/../+stdlib/remove." + mexext))
+import matlab.unittest.constraints.IsFile
+tc.assertThat(fileparts(mfilename("fullpath")) + "/../+stdlib/remove." + mexext, IsFile)
 
 d = tc.createTemporaryFolder();
 
@@ -48,7 +45,8 @@ function test_remove_empty_dir(tc)
 
 
 function test_remove_recursive(tc)
-tc.assumeTrue(isfile(fileparts(mfilename("fullpath")) + "/../+stdlib/remove." + mexext))
+import matlab.unittest.constraints.IsFile
+tc.assertThat(fileparts(mfilename("fullpath")) + "/../+stdlib/remove." + mexext, IsFile)
 
 d = tc.createTemporaryFolder();