Feature/junit report

CMakeLists.txt

@@ -56,7 +56,13 @@ set(SourceFiles
     Source/tests/BusTests.cpp
     Source/tests/ParameterFuzzTests.cpp
     Source/TestUtilities.cpp
-    Source/Validator.cpp)
+    Source/Validator.cpp
+    Source/JUnitReport.cpp
+    Source/JUnitReport.h
+    Source/JUnitListener.cpp
+    Source/JUnitListener.h
+    Source/PluginTestResult.h
+    )
 
 target_sources(pluginval PRIVATE ${SourceFiles})
 source_group(TREE ${CMAKE_CURRENT_SOURCE_DIR}/Source PREFIX Source FILES ${SourceFiles})

Source/CommandLine.cpp

@@ -49,6 +49,8 @@ CommandLineValidator::CommandLineValidator()
 {
     validator.addChangeListener (this);
     validator.addListener (this);
+
+    validator.addListener(&junitListener);
 }
 
 CommandLineValidator::~CommandLineValidator()
@@ -57,6 +59,8 @@ CommandLineValidator::~CommandLineValidator()
 
 void CommandLineValidator::validate (const StringArray& fileOrIDs, PluginTests::Options options, bool validateInProcess)
 {
+    junitListener.setReportFile(options.junitReportFile);
+
     validator.setValidateInProcess (validateInProcess);
     validator.validate (fileOrIDs, options);
 }
@@ -79,7 +83,7 @@ void CommandLineValidator::logMessage (const String& m)
     std::cout << m << "\n";
 }
 
-void CommandLineValidator::itemComplete (const String& id, int numItemFailures)
+void CommandLineValidator::itemComplete (const String& id, int numItemFailures, const PluginTestResultArray&)
 {
     logMessage ("\nFinished validating: " + id);
 
@@ -217,6 +221,11 @@ StringArray getDisabledTest (const ArgumentList& args)
     return disabledTests;
 }
 
+File getJunitReportFile (const ArgumentList& args)
+{
+    return getOptionValue (args, "--junit-report-file", {}, "Missing junit-report-file path argument!").toString();
+}
+
 //==============================================================================
 String parseCommandLineArgs (String commandLine)
 {
@@ -332,6 +341,8 @@ static String getHelpMessage()
          << "    If specified, sets the list of sample rates at which tests will be executed (default=44100,48000,96000)" << newLine
          << "  --block-sizes [list of comma separated block sizes]" << newLine
          << "    If specified, sets the list of block sizes at which tests will be executed (default=64,128,256,512,1024)" << newLine
+         << "  --junit-report-file [output filename]" << newLine
+         << "    If specified, creates JUnit report" << newLine
          << "  --version" << newLine
          << "    Print pluginval version." << newLine
          << newLine
@@ -394,6 +405,7 @@ static void validate (CommandLineValidator& validator, const ArgumentList& args)
         options.disabledTests = getDisabledTest (args);
         options.sampleRates = getSampleRates (args);
         options.blockSizes = getBlockSizes (args);
+        options.junitReportFile = getJunitReportFile(args);
 
         validator.validate (fileOrIDs,
                             options,

Source/CommandLine.h

@@ -15,7 +15,9 @@
 #pragma once
 
 #include <JuceHeader.h>
+#include "PluginTestResult.h"
 #include "Validator.h"
+#include "JUnitListener.h"
 
 struct CommandLineValidator : private ChangeListener,
                               private Validator::Listener
@@ -27,14 +29,15 @@ struct CommandLineValidator : private ChangeListener,
 
 private:
     Validator validator;
+    JUnitListener junitListener;
     String currentID;
     std::atomic<int> numFailures { 0 };
 
     void changeListenerCallback (ChangeBroadcaster*) override;
 
     void validationStarted (const String&) override;
     void logMessage (const String& m) override;
-    void itemComplete (const String&, int numItemFailures) override;
+    void itemComplete (const String&, int numItemFailures, const PluginTestResultArray&) override;
     void allItemsComplete() override;
     void connectionLost() override;
 };

Source/JUnitListener.cpp

@@ -0,0 +1,39 @@
+#include "JUnitListener.h"
+
+#include "JUnitReport.h"
+
+JUnitListener::JUnitListener()
+{
+}
+
+void JUnitListener::setReportFile(const File &newReportFile)
+{
+    reportFile = newReportFile;
+}
+
+void JUnitListener::validationStarted(const String &)
+{
+}
+
+void JUnitListener::logMessage(const String &)
+{
+}
+
+void JUnitListener::itemComplete(const String &id, int, const PluginTestResultArray &itemResults)
+{
+    results.set(id, itemResults);
+}
+
+void JUnitListener::allItemsComplete()
+{
+    if (!reportFile.getFullPathName().isEmpty())
+    {
+        JUnitReport::write(results, reportFile);
+    }
+}
+
+void JUnitListener::connectionLost()
+{
+    Validator::Listener::connectionLost();
+    // TODO: create report when connection is lost
+}

Source/JUnitListener.h

@@ -0,0 +1,21 @@
+#pragma once
+
+#include <JuceHeader.h>
+#include "PluginTestResult.h"
+#include "Validator.h"
+
+class JUnitListener: public Validator::Listener
+{
+public:
+    JUnitListener();
+    void setReportFile(const File& newReportFile);
+private:
+    void validationStarted (const String& id) override;
+    void logMessage (const String& m) override;
+    void itemComplete (const String& id, int numItemFailures, const PluginTestResultArray& itemResults) override;
+    void allItemsComplete() override;
+    void connectionLost() override;
+
+    File reportFile;
+    HashMap<String, PluginTestResultArray> results;
+};

Source/JUnitReport.cpp

@@ -0,0 +1,105 @@
+#include "JUnitReport.h"
+
+namespace JUnitReport
+{
+
+XmlElement* createTestCaseElement(const String& pluginName, int testIndex, const PluginTestResult& r)
+{
+    auto testcase = new XmlElement("testcase");
+
+    testcase->setAttribute("classname", r.result.unitTestName);
+
+    // Adding test index here to have unique name for each test case execution
+#if defined(USE_FILENAME_IN_JUNIT_REPORT)
+    testcase->setAttribute("name", "Test " + String(testIndex + 1) + ": " + r.subcategoryName);
+    testcase->setAttribute("file", pluginName);
+#else
+    testcase->setAttribute("name", "Test " + String(testIndex + 1) + ": " + r.result.subcategoryName + " of " + pluginName);
+#endif
+
+    auto duration = (r.result.endTime - r.result.startTime).inMilliseconds() / 1000.0;
+    testcase->setAttribute("time", duration);
+
+    String output = r.output.joinIntoString("\n");
+    if (r.result.failures == 0)
+    {
+        auto system_out = new XmlElement("system-out");
+        system_out->addTextElement(output);
+
+        testcase->prependChildElement(system_out);
+    }
+    else
+    {
+        auto failure = new XmlElement("failure");
+        failure->setAttribute("type", "ERROR");
+        failure->setAttribute("message", r.result.messages.joinIntoString(" "));
+        failure->addTextElement(output);
+
+        testcase->prependChildElement(failure);
+    }
+    return testcase;
+}
+
+XmlElement* createTestSuiteElement(const String& pluginName)
+{
+    auto testsuite = new XmlElement("testsuite");
+    testsuite->setAttribute("package", "pluginval");
+    testsuite->setAttribute("name", "pluginval of " + pluginName + " on " + SystemStats::getOperatingSystemName());
+    return testsuite;
+}
+
+void addTestsStats(XmlElement* element, int tests, int failures, int64 duration)
+{
+    element->setAttribute("tests", tests);
+    element->setAttribute("failures", failures);
+    element->setAttribute("time", duration / 1000.0);
+}
+
+bool write(const HashMap<String, PluginTestResultArray> &allResults, File &output)
+{
+    XmlElement testsuites("testsuites");
+    testsuites.setAttribute("name", "pluginval test suites");
+
+    int total_failures = 0;
+    int total_tests = 0;
+    int64 total_duration = 0;
+    int test_index = 0;
+    for (auto it = allResults.begin(); it != allResults.end(); ++it)
+    {
+        const auto results = it.getValue();
+        auto pluginName = it.getKey();
+
+        int suite_failures = 0;
+        int64 suite_duration = 0;
+        int suite_tests = results.size();
+
+        auto testsuite = createTestSuiteElement(pluginName);
+
+        for (const auto& r: results)
+        {
+            auto testcase = createTestCaseElement(pluginName, test_index, r);
+            testsuite->prependChildElement(testcase);
+
+            // calculate totals for test suite
+            suite_failures += r.result.failures;
+            suite_duration += (r.result.endTime - r.result.startTime).inMilliseconds();
+
+            test_index++;
+        }
+
+        addTestsStats(testsuite, suite_tests, suite_failures, suite_duration);
+
+        testsuites.prependChildElement(testsuite);
+
+        // accumulate totals for all test suites
+        total_failures += suite_failures;
+        total_duration += suite_duration;
+        total_tests += suite_tests;
+    }
+
+    addTestsStats(&testsuites, total_tests, total_failures, total_duration);
+
+    return testsuites.writeTo(output);
+}
+
+} // namespace JUnitReport

Source/JUnitReport.h

@@ -0,0 +1,11 @@
+#pragma once
+
+#include <JuceHeader.h>
+#include "PluginTestResult.h"
+
+namespace JUnitReport
+{
+
+bool write(const HashMap<String, PluginTestResultArray> &allResults, File &output);
+
+} // namespace JUnitReport

Source/MainComponent.h

@@ -15,6 +15,7 @@
 #pragma once
 
 #include <JuceHeader.h>
+#include "PluginTestResult.h"
 #include "Validator.h"
 #include "CrashHandler.h"
 
@@ -91,7 +92,7 @@ struct ConnectionStatus : public Component,
     {
     }
 
-    void itemComplete (const String&, int) override
+    void itemComplete (const String&, int, const PluginTestResultArray&) override
     {
     }
 
@@ -194,7 +195,7 @@ struct ConsoleComponent : public Component,
         std::cout << m << "\n";
     }
 
-    void itemComplete (const String& id, int numFailures) override
+    void itemComplete (const String& id, int numFailures, const PluginTestResultArray&) override
     {
         logMessage ("\nFinished validating: " + id);
 

Source/PluginTestResult.h

@@ -0,0 +1,17 @@
+#pragma once
+
+#include <JuceHeader.h>
+
+struct PluginTestResult
+{
+    PluginTestResult(const UnitTestRunner::TestResult &result, const StringArray &output):
+        result(result), output(output)
+    {
+    }
+
+    UnitTestRunner::TestResult result;
+    StringArray output;
+};
+
+using PluginTestResultArray = Array<PluginTestResult>;
+

Source/PluginTests.h

@@ -38,6 +38,7 @@ struct PluginTests : public UnitTest
         StringArray disabledTests;          /**< List of disabled tests. */
         std::vector<double> sampleRates;    /**< List of sample rates. */
         std::vector<int> blockSizes;        /**< List of block sizes. */
+        File junitReportFile;               /**< JUnit report file. */
     };
 
     /** Creates a set of tests for a fileOrIdentifier. */