Merge branch 'dev' into deploy

biosimulators_test_suite/data_model.py

@@ -50,11 +50,13 @@ def __init__(self, id=None, name=None, description=None, output_medium=OutputMed
         self.output_medium = output_medium
 
     @abc.abstractmethod
-    def eval(self, specifications):
+    def eval(self, specifications, synthetic_archives_dir=None):
         """ Evaluate a simulator's performance on a test case
 
         Args:
             specifications (:obj:`dict`): specifications of the simulator to validate
+            synthetic_archives_dir (:obj:`str`, optional): Directory to save the synthetic COMBINE/OMEX archives
+                generated by the test cases
 
         Raises:
             :obj:`SkippedTestCaseException`: if the test case is not applicable to the simulator

biosimulators_test_suite/exec_cli.py

@@ -39,7 +39,11 @@ class Meta:
                       "Default: evaluate all test cases"
                 ),
             )),
-            (['-o', '--report'], dict(
+            (['--synthetic-archives-dir'], dict(
+                default=None,
+                help="Directory to save the synthetic COMBINE/OMEX archives generated by the test cases",
+            )),
+            (['--report'], dict(
                 default=None,
                 help="Path to save a report of the results in JSON format",
             )),
@@ -62,6 +66,7 @@ def _default(self):
                 args.specifications,
                 case_ids=args.case_ids,
                 verbose=args.verbose,
+                synthetic_archives_dir=args.synthetic_archives_dir,
                 output_medium=OutputMedium.console)
             results = validator.run()
 

biosimulators_test_suite/exec_core.py

@@ -23,6 +23,7 @@
 import collections
 import datetime
 import inspect
+import os
 import sys
 import termcolor
 import warnings
@@ -38,16 +39,18 @@ class SimulatorValidator(object):
         specifications (:obj:`str` or :obj:`dict`): path or URL to the specifications of the simulator, or the specifications of the simulator
         cases (:obj:`collections.OrderedDict` of :obj:`types.ModuleType` to :obj:`TestCase`): groups of test cases
         verbose (:obj:`bool`): if :obj:`True`, display stdout/stderr from executing cases in real time
+        synthetic_archives_dir (:obj:`str`): Directory to save the synthetic COMBINE/OMEX archives generated by the test cases
         output_medium (:obj:`OutputMedium`): environment where outputs will be sent
     """
 
-    def __init__(self, specifications, case_ids=None, verbose=False, output_medium=OutputMedium.console):
+    def __init__(self, specifications, case_ids=None, verbose=False, synthetic_archives_dir=None, output_medium=OutputMedium.console):
         """
         Args:
             specifications (:obj:`str` or :obj:`dict`): path or URL to the specifications of the simulator, or the specifications of the simulator
             case_ids (:obj:`list` of :obj:`str`, optional): List of ids of test cases to verify. If :obj:`ids`
                 is none, all test cases are verified.
             verbose (:obj:`bool`, optional): if :obj:`True`, display stdout/stderr from executing cases in real time
+            synthetic_archives_dir (:obj:`str`, optional): Directory to save the synthetic COMBINE/OMEX archives generated by the test cases
             output_medium (:obj:`OutputMedium`, optional): environment where outputs will be sent
         """
         # if necessary, get and validate specifications of simulator
@@ -56,6 +59,9 @@ def __init__(self, specifications, case_ids=None, verbose=False, output_medium=O
 
         self.specifications = specifications
         self.verbose = verbose
+        if synthetic_archives_dir and not os.path.isdir(synthetic_archives_dir):
+            os.makedirs(synthetic_archives_dir)
+        self.synthetic_archives_dir = synthetic_archives_dir
         self.output_medium = output_medium
 
         self.cases = self.find_cases(ids=case_ids)
@@ -225,7 +231,7 @@ def eval_case(self, case):
                 warnings.simplefilter("always", TestCaseWarning)
 
                 try:
-                    case.eval(self.specifications)
+                    case.eval(self.specifications, synthetic_archives_dir=self.synthetic_archives_dir)
                     type = TestCaseResultType.passed
                     exception = None
                     skip_reason = None

biosimulators_test_suite/test_case/cli.py

@@ -23,11 +23,13 @@
 class CliDisplaysHelpInline(TestCase):
     """ Test that a command-line interface provides inline help. """
 
-    def eval(self, specifications):
+    def eval(self, specifications, synthetic_archives_dir=None):
         """ Evaluate a simulator's performance on a test case
 
         Args:
             specifications (:obj:`dict`): specifications of the simulator to validate
+            synthetic_archives_dir (:obj:`str`, optional): Directory to save the synthetic COMBINE/OMEX archives
+                generated by the test cases
 
         Raises:
             :obj:`Exception`: if the simulator did not pass the test case
@@ -86,11 +88,13 @@ class CliDescribesSupportedEnvironmentVariablesInline(TestCase):
     simulator supports.
     """
 
-    def eval(self, specifications):
+    def eval(self, specifications, synthetic_archives_dir=None):
         """ Evaluate a simulator's performance on a test case
 
         Args:
             specifications (:obj:`dict`): specifications of the simulator to validate
+            synthetic_archives_dir (:obj:`str`, optional): Directory to save the synthetic COMBINE/OMEX archives
+                generated by the test cases
 
         Raises:
             :obj:`Exception`: if the simulator did not pass the test case
@@ -123,11 +127,13 @@ def eval(self, specifications):
 class CliDisplaysVersionInformationInline(TestCase):
     """ Test that a command-line interface provides version information inline. """
 
-    def eval(self, specifications):
+    def eval(self, specifications, synthetic_archives_dir=None):
         """ Evaluate a simulator's performance on a test case
 
         Args:
             specifications (:obj:`dict`): specifications of the simulator to validate
+            synthetic_archives_dir (:obj:`str`, optional): Directory to save the synthetic COMBINE/OMEX archives
+                generated by the test cases
 
         Raises:
             :obj:`Exception`: if the simulator did not pass the test case

biosimulators_test_suite/test_case/combine_archive.py

@@ -38,7 +38,8 @@ class CombineArchiveTestCase(SingleMasterSedDocumentCombineArchiveTestCase):
     SED_DOCUMENTS_HAVE_SAME_NAMES = False
 
     @abc.abstractmethod
-    def _is_concrete(): pass
+    def _is_concrete():
+        pass  # pragma: no cover
 
     def build_synthetic_archives(self, specifications, curated_archive, curated_archive_dir, curated_sed_docs):
         """ Generate a synthetic archive with a copy of each task and each report

biosimulators_test_suite/test_case/docker_image.py

@@ -22,11 +22,13 @@
 class DefaultUserIsRoot(TestCase):
     """ Test that the default user of a Docker image is root """
 
-    def eval(self, specifications, expected_user=(None, '', '0')):
+    def eval(self, specifications, synthetic_archives_dir=None, expected_user=(None, '', '0')):
         """ Evaluate a simulator's performance on a test case
 
         Args:
             specifications (:obj:`dict`): specifications of the simulator to validate
+            synthetic_archives_dir (:obj:`str`, optional): Directory to save the synthetic COMBINE/OMEX archives
+                generated by the test cases
             expected_user (:obj:`tuple`, optional): expected user
 
         Raises:
@@ -49,11 +51,13 @@ def eval(self, specifications, expected_user=(None, '', '0')):
 class DeclaresSupportedEnvironmentVariables(TestCase):
     """ Test if a Docker image declares the environment variables that is supports """
 
-    def eval(self, specifications):
+    def eval(self, specifications, synthetic_archives_dir=None):
         """ Evaluate a simulator's performance on a test case
 
         Args:
             specifications (:obj:`dict`): specifications of the simulator to validate
+            synthetic_archives_dir (:obj:`str`, optional): Directory to save the synthetic COMBINE/OMEX archives
+                generated by the test cases
 
         Raises:
             :obj:`Exception`: if the simulator did not pass the test case
@@ -92,11 +96,13 @@ class HasOciLabels(TestCase):
         'org.opencontainers.image.created',
     ]
 
-    def eval(self, specifications):
+    def eval(self, specifications, synthetic_archives_dir=None):
         """ Evaluate a simulator's performance on a test case
 
         Args:
             specifications (:obj:`dict`): specifications of the simulator to validate
+            synthetic_archives_dir (:obj:`str`, optional): Directory to save the synthetic COMBINE/OMEX archives
+                generated by the test cases
 
         Raises:
             :obj:`Exception`: if the simulator did not pass the test case
@@ -127,11 +133,13 @@ class HasBioContainersLabels(TestCase):
         "version",
     ]
 
-    def eval(self, specifications):
+    def eval(self, specifications, synthetic_archives_dir=None):
         """ Evaluate a simulator's performance on a test case
 
         Args:
             specifications (:obj:`dict`): specifications of the simulator to validate
+            synthetic_archives_dir (:obj:`str`, optional): Directory to save the synthetic COMBINE/OMEX archives
+                generated by the test cases
 
         Raises:
             :obj:`Exception`: if the simulator did not pass the test case

biosimulators_test_suite/test_case/published_project.py

@@ -263,11 +263,13 @@ def compatible_with_specifications(self, specifications):
 
         return True
 
-    def eval(self, specifications):
+    def eval(self, specifications, synthetic_archives_dir=None):
         """ Evaluate a simulator's performance on a test case
 
         Args:
             specifications (:obj:`dict`): specifications of the simulator to validate
+            synthetic_archives_dir (:obj:`str`, optional): Directory to save the synthetic COMBINE/OMEX archives
+                generated by the test cases
 
         Raises:
             :obj:`SkippedTestCaseException`: if the test case is not applicable to the simulator
@@ -376,7 +378,7 @@ def eval(self, specifications):
         # check expected outputs created: plots
         if os.path.isfile(os.path.join(out_dir, get_config().PLOTS_PATH)):
             archive = biosimulators_utils.archive.io.ArchiveReader().run(os.path.join(out_dir, 'plots.zip'))
-            plot_ids = set(file.archive_path for file in archive.files)
+            plot_ids = set(os.path.splitext(file.archive_path)[0] for file in archive.files)
         else:
             plot_ids = set()
 
@@ -395,10 +397,10 @@ def eval(self, specifications):
 
         if extra_plot_ids:
             if self.assert_no_extra_plots:
-                errors.append('Extra plots were not produced:\n  {}'.format(
+                errors.append('Extra plots were produced:\n  {}'.format(
                     '\n  '.join(sorted(extra_plot_ids))))
             else:
-                warnings.warn('Extra plots were not produced:\n  {}'.format(
+                warnings.warn('Extra plots were produced:\n  {}'.format(
                     '\n  '.join(sorted(extra_plot_ids))), InvalidOutputsWarning)
 
         # cleanup outputs
@@ -439,11 +441,13 @@ def __init__(self, id=None, name=None, description=None, output_medium=OutputMed
         self.published_projects_test_cases = published_projects_test_cases or []
         self._published_projects_test_case = None
 
-    def eval(self, specifications):
+    def eval(self, specifications, synthetic_archives_dir=None):
         """ Evaluate a simulator's performance on a test case
 
         Args:
             specifications (:obj:`dict`): specifications of the simulator to validate
+            synthetic_archives_dir (:obj:`str`, optional): Directory to save the synthetic COMBINE/OMEX archives
+                generated by the test cases
 
         Returns:
             :obj:`bool`: whether there were no warnings about the outputs
@@ -452,7 +456,7 @@ def eval(self, specifications):
             :obj:`Exception`: if the simulator did not pass the test case
         """
         try:
-            return_value = self._eval(specifications)
+            return_value = self._eval(specifications, synthetic_archives_dir=synthetic_archives_dir)
         except Exception as exception:
             if self.REPORT_ERROR_AS_SKIP:
                 raise SkippedTestCaseException(str(exception))
@@ -461,11 +465,13 @@ def eval(self, specifications):
 
         return return_value
 
-    def _eval(self, specifications):
+    def _eval(self, specifications, synthetic_archives_dir=None):
         """ Evaluate a simulator's performance on a test case
 
         Args:
             specifications (:obj:`dict`): specifications of the simulator to validate
+            synthetic_archives_dir (:obj:`str`, optional): Directory to save the synthetic COMBINE/OMEX archives
+                generated by the test cases
 
         Returns:
             :obj:`bool`: whether there were no warnings about the outputs
@@ -495,7 +501,7 @@ def _eval(self, specifications):
 
                 # remove manifest from contents because libSED-ML occassionally has trouble with this
                 elif (
-                    content.location == './manifest.xml'
+                    content.location in ['manifest.xml', './manifest.xml']
                     and content.format == 'http://identifiers.org/combine.specifications/omex-manifest'
                 ):
                     curated_archive.contents.remove(content)
@@ -517,14 +523,16 @@ def _eval(self, specifications):
             specifications, curated_archive, shared_archive_dir, curated_sed_docs)
         has_warnings = False
         try:
-            for expected_results_of_synthetic_archive in expected_results_of_synthetic_archives:
-                if self._eval_synthetic_archive(specifications, expected_results_of_synthetic_archive, shared_archive_dir):
+            for i_archive, expected_results_of_synthetic_archive in enumerate(expected_results_of_synthetic_archives):
+                if self._eval_synthetic_archive(specifications, expected_results_of_synthetic_archive, shared_archive_dir,
+                                                i_archive, synthetic_archives_dir=synthetic_archives_dir):
                     has_warnings = True
         finally:
             shutil.rmtree(temp_dir)
         return not has_warnings
 
-    def _eval_synthetic_archive(self, specifications, expected_results_of_synthetic_archive, shared_archive_dir):
+    def _eval_synthetic_archive(self, specifications, expected_results_of_synthetic_archive, shared_archive_dir,
+                                i_synthetic_archive, synthetic_archives_dir=None):
         synthetic_archive = expected_results_of_synthetic_archive.archive
         synthetic_sed_docs = expected_results_of_synthetic_archive.sed_documents
         is_success_expected = expected_results_of_synthetic_archive.is_success_expected
@@ -536,6 +544,20 @@ def _eval_synthetic_archive(self, specifications, expected_results_of_synthetic_
             sedml_writer.run(sed_doc, os.path.join(shared_archive_dir, location))
         CombineArchiveWriter().run(synthetic_archive, shared_archive_dir, synthetic_archive_filename)
 
+        if synthetic_archives_dir:
+            cls = self.__class__
+            module = cls.__module__.partition('biosimulators_test_suite.test_case.')[2]
+            export_synthetic_archive_dirname = os.path.join(synthetic_archives_dir, module, cls.__name__)
+            if not os.path.isdir(export_synthetic_archive_dirname):
+                os.makedirs(export_synthetic_archive_dirname)
+            export_synthetic_archive_filename = os.path.join(export_synthetic_archive_dirname,
+                                                             '{}.{}.omex'.format(
+                                                                 str(i_synthetic_archive + 1),
+                                                                 'execution-should-succeed'
+                                                                 if is_success_expected else
+                                                                 'execute-should-fail'))
+            shutil.copy(synthetic_archive_filename, export_synthetic_archive_filename)
+
         # use synthetic archive to test simulator
         outputs_dir = os.path.join(temp_dir, 'outputs')
         pull_docker_image = Config().pull_docker_image

docs-src/tutorial.rst

@@ -69,3 +69,37 @@ The test suite can also be run locally as illustrated below.
       --combine-case \
         sbml-core/Caravagna-J-Theor-Biol-2010-tumor-suppressive-oscillations
         sbml-core/Ciliberto-J-Cell-Biol-2003-morphogenesis-checkpoint
+
+Saving the synthetic COMBINE archives generated by the test cases
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
+Optionally, the ``--synthetic-archives-dir`` argument can be used to save the synthetic
+COMBINE/OMEX archives generated by the test cases to a directory. This enables developers
+to inspect how the test suite verifies simulation tools. 
+
+.. code-block:: text
+
+    # Run all tests
+    biosimulators-test-suite /path/to/simulator/specifications.json \
+      --synthetic-archives-dir /path/to/save/synthetic-archives
+
+The synethetically-generated archives will be saved to a separate directory for each test
+case in a separate directory for each module. The files will have names that indicate the
+order in which they were executed and whether simulators are expected to sucessfully execute
+the archive or not. For example, the archive generated by the 
+:py:class:`biosimulators_test_suite.test_case.sedml.SimulatorSupportsMultipleTasksPerSedDocument`
+test case will be saved to
+``/path/to/save/synthetic-archives/sedml/SimulatorSupportsMultipleTasksPerSedDocument/1.execution-should-succeed.omex``.
+
+
+Saving the results of the test cases to a file
+++++++++++++++++++++++++++++++++++++++++++++++
+
+Optionally, the ``--report`` argument can be used to save the results of the test cases
+to a JSON file.
+
+.. code-block:: text
+
+    # Run all tests
+    biosimulators-test-suite /path/to/simulator/specifications.json \
+      --report /path/to/save/results.json

examples/sbml-core/Vilar-PNAS-2002-minimal-circardian-clock-continuous.json

@@ -224,14 +224,14 @@
     }
   ],
   "expectedPlots": [{
-      "id": "simulation.sedml.sedml/plot2d_Fig_1a"
+      "id": "simulation.sedml/plot2d_Fig_1a"
     },
     {
-      "id": "simulation.sedml.sedml/plot2d_low_delta_R_det"
+      "id": "simulation.sedml/plot2d_low_delta_R_det"
     }, {
-      "id": "simulation.sedml.sedml/plot2d_Euler_default"
+      "id": "simulation.sedml/plot2d_Euler_default"
     }, {
-      "id": "simulation.sedml.sedml/plot2d_Euler_small_step_size"
+      "id": "simulation.sedml/plot2d_Euler_small_step_size"
     }
   ],
   "runtimeFailureAlertType": "exception",

examples/sbml-core/Vilar-PNAS-2002-minimal-circardian-clock-discrete.json

@@ -113,9 +113,9 @@
     }
   ],
   "expectedPlots": [{
-    "id": "simulation.sedml.sedml/plot2d_Fig_1_c"
+    "id": "simulation.sedml/plot2d_Fig_1_c"
   }, {
-    "id": "simulation.sedml.sedml/plot2d_low_delta_R_stoch"
+    "id": "simulation.sedml/plot2d_low_delta_R_stoch"
   }],
   "runtimeFailureAlertType": "exception",
   "identifiers": [{