From 4600c39a9ad19e3289f5fe68ea54f83f0fdcb6d6 Mon Sep 17 00:00:00 2001
From: Jonathan Karr <jonrkarr@gmail.com>
Date: Mon, 4 Jan 2021 02:22:27 -0500
Subject: [PATCH] making support for non-zero initial times a warning

---
 biosimulators_test_suite/_version.py          |  2 +-
 .../test_case/published_project.py            | 29 ++++++++++++++++++-
 biosimulators_test_suite/test_case/sedml.py   |  4 +++
 tests/test_case/test_sedml.py                 | 10 +++++--
 4 files changed, 41 insertions(+), 4 deletions(-)

diff --git a/biosimulators_test_suite/_version.py b/biosimulators_test_suite/_version.py
index 1c98a23..850505a 100644
--- a/biosimulators_test_suite/_version.py
+++ b/biosimulators_test_suite/_version.py
@@ -1 +1 @@
-__version__ = '0.1.9'
+__version__ = '0.1.10'
diff --git a/biosimulators_test_suite/test_case/published_project.py b/biosimulators_test_suite/test_case/published_project.py
index 1ee32b4..b06ade6 100644
--- a/biosimulators_test_suite/test_case/published_project.py
+++ b/biosimulators_test_suite/test_case/published_project.py
@@ -9,7 +9,7 @@
 from ..data_model import (TestCase, SedTaskRequirements, ExpectedSedReport, ExpectedSedPlot,
                           AlertType, OutputMedium)
 from ..exceptions import InvalidOuputsException, SkippedTestCaseException
-from ..warnings import IgnoredTestCaseWarning, SimulatorRuntimeErrorWarning, InvalidOuputsWarning
+from ..warnings import IgnoredTestCaseWarning, SimulatorRuntimeErrorWarning, InvalidOuputsWarning, TestCaseWarning
 from .utils import are_array_shapes_equivalent
 from biosimulators_utils.combine.data_model import CombineArchive, CombineArchiveContentFormatPattern  # noqa: F401
 from biosimulators_utils.combine.io import CombineArchiveReader, CombineArchiveWriter
@@ -951,6 +951,33 @@ def modify_simulation(self, simulation):
         """
         pass  # pragma: no cover
 
+    @property
+    def report_error_as_warning(self):
+        return False
+
+    def eval(self, specifications):
+        """ Evaluate a simulator's performance on a test case
+
+        Args:
+            specifications (:obj:`dict`): specifications of the simulator to validate
+
+        Returns:
+            :obj:`object`: data returned by :obj:`eval_outputs`
+
+        Raises:
+            :obj:`Exception`: if the simulator did not pass the test case
+        """
+        try:
+            return_value = super(UniformTimeCourseTestCase, self).eval(specifications)
+        except Exception as exception:
+            if self.report_error_as_warning:
+                warnings.warn(str(exception), TestCaseWarning)
+                return_value = False
+            else:
+                raise
+
+        return return_value
+
     def eval_outputs(self, specifications, synthetic_archive, synthetic_sed_docs, outputs_dir):
         """ Test that the expected outputs were created for the synthetic archive
 
diff --git a/biosimulators_test_suite/test_case/sedml.py b/biosimulators_test_suite/test_case/sedml.py
index d193787..8fa0dba 100644
--- a/biosimulators_test_suite/test_case/sedml.py
+++ b/biosimulators_test_suite/test_case/sedml.py
@@ -456,6 +456,10 @@ def modify_simulation(self, simulation):
 class SimulatorSupportsUniformTimeCoursesWithNonZeroInitialTimes(UniformTimeCourseTestCase):
     """ Test that a simulator supports multiple time courses with non-zero initial times """
 
+    @property
+    def report_error_as_warning(self):
+        return True
+
     def modify_simulation(self, simulation):
         """ Modify a simulation
 
diff --git a/tests/test_case/test_sedml.py b/tests/test_case/test_sedml.py
index c09ecc6..133109b 100644
--- a/tests/test_case/test_sedml.py
+++ b/tests/test_case/test_sedml.py
@@ -1,7 +1,7 @@
 from biosimulators_test_suite.exceptions import InvalidOuputsException
 from biosimulators_test_suite.test_case import sedml
 from biosimulators_test_suite.test_case.published_project import SimulatorCanExecutePublishedProject, SyntheticCombineArchiveTestCase
-from biosimulators_test_suite.warnings import IgnoredTestCaseWarning, InvalidOuputsWarning
+from biosimulators_test_suite.warnings import IgnoredTestCaseWarning, InvalidOuputsWarning, TestCaseWarning
 from biosimulators_utils.archive.data_model import Archive, ArchiveFile
 from biosimulators_utils.archive.io import ArchiveWriter
 from biosimulators_utils.config import get_config
@@ -374,6 +374,11 @@ def test_SimulatorSupportsUniformTimeCoursesWithNonZeroInitialTimes(self):
             published_projects_test_cases=[curated_case])
         self.assertTrue(case.eval(specs))
 
+        with mock.patch('biosimulators_utils.simulator.exec.exec_sedml_docs_in_archive_with_containerized_simulator',
+                        side_effect=Exception('Simulation failed')):
+            with self.assertWarns(TestCaseWarning):
+                self.assertFalse(case.eval(specs))
+
     def test_SimulatorProducesLinear2DPlots_eval_outputs(self):
         case = sedml.SimulatorProducesLinear2DPlots()
 
@@ -541,7 +546,8 @@ def test_SimulatorProducesReportsWithCuratedNumberOfDimensions(self):
         })
         self.assertTrue(case.is_curated_sed_algorithm_suitable_for_building_synthetic_archive(specs, alg))
 
-        with mock.patch.object(SyntheticCombineArchiveTestCase, 'is_curated_sed_algorithm_suitable_for_building_synthetic_archive', return_value=False):
+        with mock.patch.object(SyntheticCombineArchiveTestCase,
+                               'is_curated_sed_algorithm_suitable_for_building_synthetic_archive', return_value=False):
             self.assertFalse(case.is_curated_sed_algorithm_suitable_for_building_synthetic_archive(specs, alg))
 
         # eval_outputs