Added test/level description window and plugin list searching. Thanks Jim! (#158)

* Fix typos in command-line argument descriptions

* Add custom LookAndFeel and strictness info popup

* Add `--strictness-help` command and improvements to UI

* Add plugin name filtering and enhance UI with tabbed styles

* Adjust table column widths and refine search box spacing

* Show plugin file path in list

---------

Co-authored-by: Jim Credland <[email protected]>

Author: drowaudio

Source/CommandLine.cpp

@@ -15,6 +15,7 @@
 #include "CommandLine.h"
 #include "Validator.h"
 #include "CrashHandler.h"
+#include "PluginTests.h"
 
 #if JUCE_MAC
  #include <signal.h>
@@ -322,106 +323,131 @@ static juce::StringArray mergeEnvironmentVariables (juce::StringArray args, std:
 static juce::String getHelpMessage()
 {
     const juce::String appName (juce::JUCEApplication::getInstance()->getApplicationName());
-
-    auto& newLine = juce::newLine;
-    juce::String help;
-    help << "//==============================================================================" << newLine
-         << appName << newLine
-         << juce::SystemStats::getJUCEVersion() << newLine
-         << newLine
-         << "Description: " << newLine
-         << "  Validate plugins to test compatibility with hosts and verify plugin API conformance" << newLine << newLine
-         << "Usage: "
-         << newLine
-         // commands
-         << "  --version" << newLine
-         << "    Print pluginval version." << newLine
-         << "  --validate [pathToPlugin]" << newLine
-         << "    Validates the plugin at the given path." << newLine
-         << "    N.B. the \"--validate\" flag is optional if the path is the last argument." << newLine
-         << "    This enables you to validate a plugin with simply \"pluginval path_to_plugin\"." << newLine
-         << newLine
-         // what to test
-         << "  --sample-rates [list of comma separated sample rates]" << newLine
-         << "    If specified, sets the list of sample rates at which tests will be executed" << newLine
-         << "    (default=44100,48000,96000)" << newLine
-         << "  --block-sizes [list of comma separated block sizes]" << newLine
-         << "    If specified, sets the list of block sizes at which tests will be executed" << newLine
-         << "    (default=64,128,256,512,1024)" << newLine
-         << "  --random-seed [hex or int]" << newLine
-         << "    Sets the random seed to use for the tests. Useful for replicating test" << newLine
-         << "    environments." << newLine
-         << "  --data-file [pathToFile]" << newLine
-         << "    If specified, sets a path to a data file which can be used by tests to" << newLine
-         << "    configure themselves. This can be useful for things like known audio output." << newLine
-         << newLine
-         // how to test
-         << "  --strictness-level [1-10]" << newLine
-         << "    Sets the strictness level to use. A minimum level of 5 (also the default)" << newLine
-         << "    is recomended for compatibility." << newLine
-         << "    Higher levels include longer, more thorough tests such as fuzzing." << newLine
-         << "  --timeout-ms [numMilliseconds]" << newLine
-         << "    Sets a timout which will stop validation with an error if no output from any" << newLine
-         << "    test has happened for this number of ms." << newLine
-         << "    By default this is 30s but can be set to \"-1\" (must be quoted) to never timeout." << newLine
-         << "  --rtcheck [empty, disabled, enabled or relaxed]" << newLine
-         << "    Turns on real-time saftey checks using rtcheck (macOS and Linux only)." << newLine
-         << "    relaxed mode doesn't run the checks for the first processing block as a lot of plugins" << newLine
-         << "    use this to allocate or initialise thread-locals (which can allocate)" << newLine
-         << newLine
-         // repeating tests
-         << "  --repeat [num repeats]" << newLine
-         << "    If specified repeats the tests a given number of times. Note that this does" << newLine
-         << "    not delete and re-instantiate the plugin for each repeat." << newLine
-         << "  --randomise" << newLine
-         << "    If specified, the tests are run in a random order per repeat." << newLine
-         << newLine
-         // test selection
-         << "  --skip-gui-tests" << newLine
-         << "    If specified, avoids tests that create GUI windows, which can cause problems" << newLine
-         << "    on headless CI systems." << newLine
-         << "  --disabled-tests [pathToFile]" << newLine
-         << "    If specified, sets a path to a file that should have the names of disabled" << newLine
-         << "    tests on each row." << newLine
-         // external validators
-         << "  --vst3validator [pathToValidator]" << newLine
-         << "    If specified, this will run the VST3 validator as part of the test process." << newLine
-         << newLine
-         // output
-         << "  --output-dir [pathToDir]" << newLine
-         << "    If specified, sets a directory to store the log files. This can be useful" << newLine
-         << "    for continuous integration." << newLine
-         << "  --output-filename [filename]" << newLine
-         << "    If specified, sets a filename for the log files (within 'output-dir' or" << newLine
-         << "    (lacking that) the current directory." << newLine
-         << "    By default, the name is constructed from the plugin metainformation" << newLine
-         << "  --verbose" << newLine
-         << "    If specified, outputs additional logging information. It can be useful to" << newLine
-         << "    turn this off when building with CI to avoid huge log files." << newLine
-
-         // end of options
-         << newLine
-         << "Exit code: "
-         << newLine
-         << "  0 if all tests complete successfully" << newLine
-         << "  1 if there are any errors" << newLine
-         << newLine
-         << "Additionally, you can specify any of the command line options as environment" << newLine
-         << "variables by removing prefix dashes, converting internal dashes to underscores" << newLine
-         << "and capitalising all letters, a.g." << newLine
-         << "    \"--skip-gui-tests\" > \"SKIP_GUI_TESTS=1\"" << newLine
-         << "    \"--timeout-ms 30000\" > \"TIMEOUT_MS=30000\"" << newLine
-         << "Specifying specific command-line options will override any environment variables" << newLine
-         << "set for that option." << newLine;
-
-    return help;
+    const juce::String juceVersion (juce::SystemStats::getJUCEVersion());
+
+    return juce::String (R"(//==============================================================================
+)" + appName + R"(
+)" + juceVersion + R"(
+
+Description:
+  Validate plugins to test compatibility with hosts and verify plugin API conformance
+
+Usage:
+  --version
+    Print pluginval version.
+  --validate [pathToPlugin]
+    Validates the plugin at the given path.
+    N.B. the "--validate" flag is optional if the path is the last argument.
+    This enables you to validate a plugin with simply "pluginval path_to_plugin".
+
+  --sample-rates [list of comma separated sample rates]
+    If specified, sets the list of sample rates at which tests will be executed
+    (default=44100,48000,96000)
+  --block-sizes [list of comma separated block sizes]
+    If specified, sets the list of block sizes at which tests will be executed
+    (default=64,128,256,512,1024)
+  --random-seed [hex or int]
+    Sets the random seed to use for the tests. Useful for replicating test
+    environments.
+  --data-file [pathToFile]
+    If specified, sets a path to a data file which can be used by tests to
+    configure themselves. This can be useful for things like known audio output.
+
+  --strictness-level [1-10]
+    Sets the strictness level to use. A minimum level of 5 (also the default)
+    is recommended for compatibility.
+    Higher levels include longer, more thorough tests such as fuzzing.
+  --strictness-help [level]
+    Lists all tests that run at the given strictness level (default: 5).
+  --timeout-ms [numMilliseconds]
+    Sets a timout which will stop validation with an error if no output from any
+    test has happened for this number of ms.
+    By default this is 30s but can be set to "-1" (must be quoted) to never timeout.
+  --rtcheck [empty, disabled, enabled or relaxed]
+    Turns on real-time safety checks using rtcheck (macOS and Linux only).
+    relaxed mode doesn't run the checks for the first processing block as a lot of plugins
+    use this to allocate or initialise thread-locals (which can allocate)
+
+  --repeat [num repeats]
+    If specified repeats the tests a given number of times. Note that this does
+    not delete and re-instantiate the plugin for each repeat.
+  --randomise
+    If specified, the tests are run in a random order per repeat.
+
+  --skip-gui-tests
+    If specified, avoids tests that create GUI windows, which can cause problems
+    on headless CI systems.
+  --disabled-tests [pathToFile]
+    If specified, sets a path to a file that should have the names of disabled
+    tests on each row.
+  --vst3validator [pathToValidator]
+    If specified, this will run the VST3 validator as part of the test process.
+
+  --output-dir [pathToDir]
+    If specified, sets a directory to store the log files. This can be useful
+    for continuous integration.
+  --output-filename [filename]
+    If specified, sets a filename for the log files (within 'output-dir' or
+    (lacking that) the current directory.
+    By default, the name is constructed from the plugin metainformation
+  --verbose
+    If specified, outputs additional logging information. It can be useful to
+    turn this off when building with CI to avoid huge log files.
+
+Exit code:
+  0 if all tests complete successfully
+  1 if there are any errors
+
+Additionally, you can specify any of the command line options as environment
+variables by removing prefix dashes, converting internal dashes to underscores
+and capitalising all letters, a.g.
+    "--skip-gui-tests" > "SKIP_GUI_TESTS=1"
+    "--timeout-ms 30000" > "TIMEOUT_MS=30000"
+Specifying specific command-line options will override any environment variables
+set for that option.
+)");
 }
 
 static juce::String getVersionText()
 {
     return juce::String ("pluginval") + " - " + VERSION;
 }
 
+static void printStrictnessHelp (int level)
+{
+    level = juce::jlimit (1, 10, level);
+
+    std::cout << "Tests at strictness level " << level << ":\n\n";
+
+    auto& allTests = PluginTest::getAllTests();
+
+    std::vector<PluginTest*> sortedTests;
+    for (auto* test : allTests)
+        sortedTests.push_back (test);
+
+    std::sort (sortedTests.begin(), sortedTests.end(),
+               [] (const PluginTest* a, const PluginTest* b)
+               { return a->strictnessLevel < b->strictnessLevel; });
+
+    for (auto* test : sortedTests)
+    {
+        if (test->strictnessLevel > level)
+            continue;
+
+        auto descriptions = test->getDescription (level);
+
+        for (const auto& desc : descriptions)
+        {
+            std::cout << "  " << desc.title;
+            if (desc.description.isNotEmpty())
+                std::cout << ": " << desc.description;
+            std::cout << "\n";
+        }
+    }
+
+    std::cout << std::endl;
+}
+
 static int getNumTestFailures (juce::UnitTestRunner& testRunner)
 {
     int numFailures = 0;
@@ -504,6 +530,17 @@ static void performCommandLine (CommandLineValidator& validator, const juce::Arg
                       "--run-tests",
                       "Runs the internal unit tests.", juce::String(),
                       [] (const auto&) { runUnitTests(); }});
+    cli.addCommand ({ "--strictness-help",
+                      "--strictness-help [level]",
+                      "Lists all tests that run at the given strictness level.", juce::String(),
+                      [] (const auto& args)
+                      {
+                          int level = 5;
+                          auto arg = getArgumentAfterOption (args, "--strictness-help");
+                          if (arg.text.isNotEmpty() && ! arg.isShortOption() && ! arg.isLongOption())
+                              level = arg.text.getIntValue();
+                          printStrictnessHelp (level);
+                      }});
 
     if (const auto retValue = cli.findAndRunCommand (args); retValue != 0)
     {
@@ -528,7 +565,8 @@ bool shouldPerformCommandLine (const juce::String& commandLine)
     return args.containsOption ("--help|-h")
         || args.containsOption ("--version")
         || args.containsOption ("--validate")
-        || args.containsOption ("--run-tests");
+        || args.containsOption ("--run-tests")
+        || args.containsOption ("--strictness-help");
 }
 
 //==============================================================================

Source/Main.cpp

@@ -16,6 +16,7 @@
 #include "MainComponent.h"
 #include "Validator.h"
 #include "CommandLine.h"
+#include "PluginvalLookAndFeel.h"
 
 //==============================================================================
 class PluginValidatorApplication  : public juce::JUCEApplication,
@@ -50,6 +51,8 @@ class PluginValidatorApplication  : public juce::JUCEApplication,
         testRunner.runTestsInCategory ("pluginval");
        #endif
 
+        juce::LookAndFeel::setDefaultLookAndFeel (&lookAndFeel);
+
         validator = std::make_unique<Validator>();
         propertiesFile.reset (getPropertiesFile());
         mainWindow = std::make_unique<MainWindow> (*validator, getApplicationName() + " v" + getApplicationVersion());
@@ -59,6 +62,7 @@ class PluginValidatorApplication  : public juce::JUCEApplication,
     {
         mainWindow.reset();
         validator.reset();
+        juce::LookAndFeel::setDefaultLookAndFeel (nullptr);
         juce::Logger::setCurrentLogger (nullptr);
     }
 
@@ -119,6 +123,7 @@ class PluginValidatorApplication  : public juce::JUCEApplication,
     };
 
 private:
+    PluginvalLookAndFeel lookAndFeel;
     std::unique_ptr<Validator> validator;
     std::unique_ptr<juce::PropertiesFile> propertiesFile;
     std::unique_ptr<MainWindow> mainWindow;