various small improvements (#119)

* Add newLines to usage-information

Closes: #116

* group flags in usage-text by functionality

make sure that the lines are no longer than 80chars
and fix some typos

* Use ISO-8601 as the time-format for logfiles

Related: #117

* Allow specifying the output-filename

Closes: #117

* Print "SUCCESS" after successully validating a plugin

Closes: #118

Author: umlaeute

Source/CommandLine.cpp

@@ -178,6 +178,11 @@ namespace
         return getOptionValue (args, "--output-dir", {}, "Missing output-dir path argument!").toString();
     }
 
+    juce::String getOutputFilename (const juce::ArgumentList& args)
+    {
+        return getOptionValue (args, "--output-filename", {}, "Missing output-filename path argument!").toString();
+    }
+
     std::vector<double> getSampleRates (const juce::ArgumentList& args)
     {
         juce::StringArray input = juce::StringArray::fromTokens (getOptionValue (args,
@@ -272,6 +277,7 @@ static Option possibleOptions[] =
     { "--skip-gui-tests",       false   },
     { "--data-file",            true    },
     { "--output-dir",           true    },
+    { "--output-filename",      true    },
     { "--repeat",               true    },
     { "--randomise",            false   },
     { "--sample-rates",         true    },
@@ -323,48 +329,82 @@ static juce::String getHelpMessage()
          << "  Validate plugins to test compatibility with hosts and verify plugin API conformance" << newLine << newLine
          << "Usage: "
          << newLine
+         // commands
+         << "  --version" << newLine
+         << "    Print pluginval version." << newLine
          << "  --validate [pathToPlugin]" << newLine
          << "    Validates the plugin at the given path." << newLine
-         << "    N.B. the --validate flag is optional if the path is the last argument. This enables you to validate a plugin with simply \"pluginval path_to_plugin\"." << newLine
-         << "  --strictness-level [1-10]" << newLine
-         << "    Sets the strictness level to use. A minimum level of 5 (also the default) is recomended for compatibility. Higher levels include longer, more thorough tests such as fuzzing." << newLine
+         << "    N.B. the \"--validate\" flag is optional if the path is the last argument." << newLine
+         << "    This enables you to validate a plugin with simply \"pluginval path_to_plugin\"." << newLine
+         << newLine
+         // what to test
+         << "  --sample-rates [list of comma separated sample rates]" << newLine
+         << "    If specified, sets the list of sample rates at which tests will be executed" << newLine
+         << "    (default=44100,48000,96000)" << newLine
+         << "  --block-sizes [list of comma separated block sizes]" << newLine
+         << "    If specified, sets the list of block sizes at which tests will be executed" << newLine
+         << "    (default=64,128,256,512,1024)" << newLine
          << "  --random-seed [hex or int]" << newLine
-         << "    Sets the random seed to use for the tests. Useful for replicating test environments." << newLine
+         << "    Sets the random seed to use for the tests. Useful for replicating test" << newLine
+         << "    environments." << newLine
+         << "  --data-file [pathToFile]" << newLine
+         << "    If specified, sets a path to a data file which can be used by tests to" << newLine
+         << "    configure themselves. This can be useful for things like known audio output." << newLine
+         << newLine
+         // how to test
+         << "  --strictness-level [1-10]" << newLine
+         << "    Sets the strictness level to use. A minimum level of 5 (also the default)" << newLine
+         << "    is recomended for compatibility." << newLine
+         << "    Higher levels include longer, more thorough tests such as fuzzing." << newLine
          << "  --timeout-ms [numMilliseconds]" << newLine
-         << "    Sets a timout which will stop validation with an error if no output from any test has happened for this number of ms." << newLine
+         << "    Sets a timout which will stop validation with an error if no output from any" << newLine
+         << "    test has happened for this number of ms." << newLine
          << "    By default this is 30s but can be set to -1 to never timeout." << newLine
-         << "  --verbose" << newLine
-         << "    If specified, outputs additional logging information. It can be useful to turn this off when building with CI to avoid huge log files." << newLine
-         << "  --skip-gui-tests" << newLine
-         << "    If specified, avoids tests that create GUI windows, which can cause problems on headless CI systems." << newLine
+         << newLine
+         // repeating tests
          << "  --repeat [num repeats]" << newLine
-         << "    If specified repeats the tests a given number of times. Note that this does not delete and re-instantiate the plugin for each repeat."
+         << "    If specified repeats the tests a given number of times. Note that this does" << newLine
+         << "    not delete and re-instantiate the plugin for each repeat." << newLine
          << "  --randomise" << newLine
-         << "    If specified the tests are run in a random order per repeat."
-         << "  --data-file [pathToFile]" << newLine
-         << "    If specified, sets a path to a data file which can be used by tests to configure themselves. This can be useful for things like known audio output." << newLine
-         << "  --output-dir [pathToDir]" << newLine
-         << "    If specified, sets a directory to store the log files. This can be useful for continuous integration." << newLine
+         << "    If specified, the tests are run in a random order per repeat." << newLine
+         << newLine
+         // test selection
+         << "  --skip-gui-tests" << newLine
+         << "    If specified, avoids tests that create GUI windows, which can cause problems" << newLine
+         << "    on headless CI systems." << newLine
          << "  --disabled-tests [pathToFile]" << newLine
-         << "    If specified, sets a path to a file that should have the names of disabled tests on each row." << newLine
-         << "  --sample-rates [list of comma separated sample rates]" << newLine
-         << "    If specified, sets the list of sample rates at which tests will be executed (default=44100,48000,96000)" << newLine
-         << "  --block-sizes [list of comma separated block sizes]" << newLine
-         << "    If specified, sets the list of block sizes at which tests will be executed (default=64,128,256,512,1024)" << newLine
+         << "    If specified, sets a path to a file that should have the names of disabled" << newLine
+         << "    tests on each row." << newLine
+         // external validators
          << "  --vst3validator [pathToValidator]" << newLine
          << "    If specified, this will run the VST3 validator as part of the test process." << newLine
-         << "  --version" << newLine
-         << "    Print pluginval version." << newLine
+         << newLine
+         // output
+         << "  --output-dir [pathToDir]" << newLine
+         << "    If specified, sets a directory to store the log files. This can be useful" << newLine
+         << "    for continuous integration." << newLine
+         << "  --output-filename [filename]" << newLine
+         << "    If specified, sets a filename for the log files (within 'output-dir' or" << newLine
+         << "    (lacking that) the current directory." << newLine
+         << "    By default, the name is constructed from the plugin metainformation" << newLine
+         << "  --verbose" << newLine
+         << "    If specified, outputs additional logging information. It can be useful to" << newLine
+         << "    turn this off when building with CI to avoid huge log files." << newLine
+
+         // end of options
          << newLine
          << "Exit code: "
          << newLine
          << "  0 if all tests complete successfully" << newLine
          << "  1 if there are any errors" << newLine
          << newLine
-         << "Additionally, you can specify any of the command line options as environment varibles by removing prefix dashes,"
-            " converting internal dashes to underscores and captialising all letters e.g. \"--skip-gui-tests\" > \"SKIP_GUI_TESTS=1\","
-            " \"--timeout-ms 30000\" > \"TIMEOUT_MS=30000\"" << newLine
-         << "Specifying specific command-line options will override any environment variables set for that option." << newLine;
+         << "Additionally, you can specify any of the command line options as environment" << newLine
+         << "variables by removing prefix dashes, converting internal dashes to underscores" << newLine
+         << "and capitalising all letters, a.g." << newLine
+         << "    \"--skip-gui-tests\" > \"SKIP_GUI_TESTS=1\"" << newLine
+         << "    \"--timeout-ms 30000\" > \"TIMEOUT_MS=30000\"" << newLine
+         << "Specifying specific command-line options will override any environment variables" << newLine
+         << "set for that option." << newLine;
 
     return help;
 }
@@ -504,6 +544,7 @@ std::pair<juce::String, PluginTests::Options> parseCommandLine (const juce::Argu
     options.randomiseTestOrder  = args.containsOption ("--randomise");
     options.dataFile            = getDataFile (args);
     options.outputDir           = getOutputDir (args);
+    options.outputFilename      = getOutputFilename (args);
     options.withGUI             = ! args.containsOption ("--skip-gui-tests");
     options.disabledTests       = getDisabledTests (args);
     options.sampleRates         = getSampleRates (args);
@@ -550,6 +591,9 @@ juce::StringArray createCommandLine (juce::String fileOrID, PluginTests::Options
     if (options.outputDir != defaults.outputDir)
         args.addArray ({ "--output-dir", options.outputDir.getFullPathName() });
 
+    if (options.outputFilename != defaults.outputFilename)
+        args.addArray ({ "--output-filename", options.outputFilename });
+
     if (options.disabledTests != defaults.disabledTests)
         args.addArray ({ "--disabled-tests", options.disabledTests.joinIntoString (",") });
 

Source/PluginTests.h

@@ -35,6 +35,7 @@ struct PluginTests : public juce::UnitTest
         bool withGUI = true;                /**< Whether or not avoid tests that instantiate a gui. */
         juce::File dataFile;                      /**< juce::File which tests can use to run user provided data. */
         juce::File outputDir;                     /**< Directory in which to write the log files for each test run. */
+        juce::String outputFilename;        /**< Filename to write logs into */
         juce::StringArray disabledTests;          /**< List of disabled tests. */
         std::vector<double> sampleRates;    /**< List of sample rates. */
         std::vector<int> blockSizes;        /**< List of block sizes. */

Source/Validator.cpp

@@ -119,23 +119,30 @@ static juce::String getFileNameFromDescription (PluginTests& test)
         return "pluginval Log";
     };
 
-    return getBaseName() + "_" + juce::Time::getCurrentTime().toString (true, true).replace (":", ",") + ".txt";
+    return getBaseName() + "_" + juce::Time::getCurrentTime().toISO8601 (false).replace (":", ",") + ".txt";
 }
 
 static juce::File getDestinationFile (PluginTests& test)
 {
     const auto dir = test.getOptions().outputDir;
+    const auto filename = test.getOptions().outputFilename;
 
     if (dir == juce::File())
-        return {};
+    {
+        if (juce::String() == filename)
+            return {};
+        return juce::File(filename);
+    }
 
     if (dir.existsAsFile() || ! dir.createDirectory())
     {
         jassertfalse;
         return {};
     }
 
-    return dir.getChildFile (getFileNameFromDescription (test));
+    if (juce::String() == filename)
+        return dir.getChildFile (getFileNameFromDescription (test));
+    return dir.getChildFile (filename);
 }
 
 static std::unique_ptr<juce::FileOutputStream> createDestinationFileStream (PluginTests& test)
@@ -162,7 +169,8 @@ static void updateFileNameIfPossible (PluginTests& test, PluginsUnitTestRunner&
     if (auto os = runner.getOutputFileStream())
     {
         const auto sourceFile = os->getFile();
-        const auto destName = getFileNameFromDescription (test);
+        const auto filename = test.getOptions().outputFilename;
+        const auto destName = (filename == juce::String())?getFileNameFromDescription (test) : filename;
 
         if (destName.isEmpty() || sourceFile.getFileName() == destName)
             return;
@@ -176,6 +184,8 @@ static void updateFileNameIfPossible (PluginTests& test, PluginsUnitTestRunner&
 
 //==============================================================================
 //==============================================================================
+inline int getNumFailures (juce::Array<juce::UnitTestRunner::TestResult> results);
+
 inline juce::Array<juce::UnitTestRunner::TestResult> runTests (PluginTests& test, std::function<void (const juce::String&)> callback)
 {
     const auto options = test.getOptions();
@@ -191,6 +201,11 @@ inline juce::Array<juce::UnitTestRunner::TestResult> runTests (PluginTests& test
         results.add (*testRunner.getResult (i));
 
     updateFileNameIfPossible (test, testRunner);
+    const int failures = getNumFailures (results);
+    if (!failures)
+        testRunner.logMessage("SUCCESS");
+    else
+        testRunner.logMessage("FAILURE");
 
     return results;
 }