Merge pull request #554 from PEtab-dev/release/2.0.0

Add PEtab 2.0 draft

.github/ISSUE_TEMPLATE/petab-extensions.md

@@ -0,0 +1,27 @@
+---
+
+name: PEtab Extension
+about: Suggest a new extension for PEtab core
+title: ''
+labels: file format
+assignees: ''
+
+---
+
+**Name of the Extension**
+Please make sure that the extension name matches the regular expression `^[a-zA-Z_][A-Za-z0-9_-]*$`.
+
+**Which problem would you like to address?**
+A clear and concise description of which use case you want to address and, if applicable, why the current specifications do not fulfill your requirements.
+
+**Describe the solution you would like**
+A clear and concise description of the changes you want to propose. Please describe any additional fields / files you would want to add, including allowed inputs and implications.
+
+**Describe why this should not be implemented by changes to PEtab core**
+A clear and concise description in what way the proposed changes introduce features that are orthogonal to the PEtab core specification.
+
+**List the extension library that implements validation checks**
+A link to the website or GitHub repository that accompanies the proposed extension.
+
+**List the toolboxes that support the proposed standard**
+A link to the website or GitHub repository that contains the software that implements support for the standard.

.rtd_pip_reqs.txt

@@ -36,3 +36,4 @@ sphinxcontrib-qthelp==1.0.7
 sphinxcontrib-serializinghtml==1.1.10
 urllib3==2.2.1
 wheel==0.43.0
+sphinx-reredirects==0.1.4

CHANGELOG.md

@@ -14,6 +14,13 @@ available at https://github.com/PEtab-dev/libpetab-python/.
 * Update tutorial.rst (#512)
 * Update how-to-cite (Closes #432) (#509)
 
+## 0.2 series
+
+### 0.2.0
+
+* Specify how PEtab functionality can be expanded through extensions.
+* YAML files are now required for the specification of PEtab problems.
+
 ## 0.1 series
 
 ### 0.1.14

README.md

@@ -23,7 +23,7 @@ for example:
 
   - Specifying multiple simulation conditions with potentially shared parameters
 
-![PEtab files](doc/gfx/petab_files.png)
+![PEtab files](doc/v1/gfx/petab_files.png)
 
 ## Documentation
 
@@ -140,6 +140,8 @@ will have to:
 
 1. Create a parameter table.
 
+1. Create a YAML file that lists the model and all of the tables above.
+
 If you are using Python, some handy functions of the
 [PEtab library](https://github.com/PEtab-dev/libpetab-python/) can help
 you with that. This includes also a PEtab validator called `petablint` which

doc/_static/petab_schema.yaml

@@ -6,8 +6,13 @@ description: PEtab parameter estimation problem config file schema
 properties:
 
   format_version:
-    type: integer
-    description: Version of the PEtab format (e.g. 1).
+    anyof:
+      - type: string
+        #  (corresponding to PEP 440).
+        pattern: ^([1-9][0-9]*!)?(0|[1-9][0-9]*)(\.(0|[1-9][0-9]*))*((a|b|rc)(0|[1-9][0-9]*))?(\.post(0|[1-9][0-9]*))?(\.dev(0|[1-9][0-9]*))?$
+      - type: integer
+
+    description: Version of the PEtab format
 
   parameter_file:
     oneOf:
@@ -17,7 +22,6 @@ properties:
       File name (absolute or relative) or URL to PEtab parameter table
       containing parameters of all models listed in `problems`. A single
       table may be split into multiple files and described as an array here.
-
   problems:
     type: array
     description: |
@@ -34,13 +38,26 @@ properties:
         files and optional visualization files.
       properties:
 
-        sbml_files:
-          type: array
-          description: List of PEtab SBML files.
-
-          items:
-            type: string
-            description: PEtab SBML file name or URL.
+        model_files:
+          type: object
+          description: One or multiple models
+
+          # the model ID
+          patternProperties:
+            "^[a-zA-Z_]\\w*$":
+              type: object
+              properties:
+                location:
+                  type: string
+                  description: Model file name or URL
+                language:
+                  type: string
+                  description: |
+                    Model language, e.g., 'sbml', 'cellml', 'bngl', 'pysb'
+              required:
+                - location
+                - language
+          additionalProperties: false
 
         measurement_files:
           type: array
@@ -74,12 +91,46 @@ properties:
             type: string
             description: PEtab visualization file name or URL.
 
+        mapping_files:
+          type: array
+          description: List of PEtab mapping files.
+
+          items:
+            type: string
+            description: PEtab mapping file name or URL.
+
       required:
-        - sbml_files
+        - model_files
         - observable_files
         - measurement_files
         - condition_files
 
+  extensions:
+    type: object
+    description: |
+      PEtab extensions being used.
+    patternProperties:
+      "^[a-zA-Z][\\-\\w]*$":
+
+        type: object
+        description: |
+          Information on a specific extension
+        properties:
+          version:
+            type: string
+            pattern: ^([1-9][0-9]*!)?(0|[1-9][0-9]*)(\.(0|[1-9][0-9]*))*((a|b|rc)(0|[1-9][0-9]*))?(\.post(0|[1-9][0-9]*))?(\.dev(0|[1-9][0-9]*))?$
+          required:
+            type: bool
+            description: |
+              Indicates whether the extension is required for the
+              mathematical interpretation of the problem.
+        required:
+          - version
+          - required
+      additionalProperties: true
+
+    additionalProperties: false
+
 required:
   - format_version
   - parameter_file

doc/conf.py

@@ -40,6 +40,7 @@
     'm2r2',
     'sphinx.ext.autosummary',
     'sphinx_markdown_tables',
+    'sphinx_reredirects',
 ]
 
 # Add any paths that contain templates here, relative to this directory.
@@ -71,6 +72,11 @@
     '.md': 'markdown',
 }
 
+# Redirects for moved files
+redirects = {
+    "documentation_data_format": "v1/documentation_data_format.html",
+}
+
 # -- Options for HTML output -------------------------------------------------
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for

doc/development.rst

@@ -192,6 +192,58 @@ Upon a new release, the PEtab editors ensure that
 * the new version of the specifications is deposited at Zenodo
 * the new release is announced on the PEtab mailing list
 
+PEtab Extensions
+----------------
+
+An elaborate, monolithic format would make it difficult to understand and
+implement support for PEtab, leading to a steep learning curve and discouraging
+support in new toolboxes. To address this issue, the PEtab format is modular and
+permits modifications through extensions that complement the core standard.
+This modular specification evens the learning curve and provides toolbox
+developers with more guidance on which features to implement to maximize
+support for real world applications. Moreover, such modular extensions
+facilitate and promote the use of specialized tools for specific, non-parameter
+estimation tasks such as visualization.
+
+Requirements for new extensions:
+
+* Specifications in PEtab extensions take precedence over PEtab core, i.e., they
+  can ease or refine format restrictions imposed by PEtab core.
+* PEtab extensions should extend PEtab core with new orthogonal features or
+  tasks, i.e., they should not make trivial changes to PEtab core.
+* PEtab extensions must be named according to ``^[a-zA-Z][a-zA-Z0-9_\-]*$``.
+* PEtab extensions must be versioned using semantic versioning.
+* If a PEtab extension changes the mathematical interpretation of a problem,
+    it must appear in the PEtab YAML file.
+* There is at least one tool that supports the proposed extension.
+* The authors provide a library that provides test cases and implements
+  validation checks for the proposed format.
+
+It is encouraged that (potential) extensions are informally discussed with the
+community as early as possible.
+Developers are free to develop any PEtab extension. To become an official
+PEtab extension, it needs to go through the following process.
+
+#. The developers write a proposal describing the motivation and specification
+   of the extension, following the respective issue template provided in this
+   repository.
+#. The proposal is submitted as an issue in this repository.
+#. The technical specification and documentation of the extension is submitted
+   as a pull request in this repository that references the respective issue.
+
+The PEtab editors jointly decide whether an extension meets the requirements
+described here. In case of a positive evaluation, they announce a poll for the
+acceptance as official extension to the PEtab forum. All members of the PEtab
+community are eligible to vote. If at least 50% of the votes are in favor,
+the extension is accepted and the respective pull requests with specifications,
+documentation and test cases are merged. There is no quorum number of votes
+for acceptance.
+
+It is encouraged that extensions are informally discussed with the community
+before initiating the process of becoming an official extension. Such
+discussions can be conducted through the communication channels mentioned
+above.
+
 Versioning of the PEtab format
 ------------------------------
 

doc/index.rst

@@ -5,7 +5,8 @@
    :maxdepth: 3
    :caption: Data format
 
-   Data format <documentation_data_format.rst>
+   PEtab v1 specs <v1/documentation_data_format.rst>
+   PEtab v2 draft <v2/documentation_data_format.rst>
    tutorial
 
 .. toctree::

doc/src/Supported functions.tsv

@@ -0,0 +1,13 @@
+Function	Comment	Argument types	Evaluates to
+``pow(a, b)``	power function `b`-th power of `a`	float, float	float
+``exp(x)``	exponential function pow(e, x);(`e` itself not a supported symbol,;but ``exp(1)`` can be used instead)	float	float
+``sqrt(x)``	square root of ``x``;``pow(x, 0.5)``	float	float
+``log(a, b)``;``log(x)``;``ln(x)``;``log2(x)``;``log10(x)``	logarithm of ``a`` with base ``b``;``log(x, e)``;``log(x, e)``;``log(x, 2)``;``log(x, 10)``;(``log(0)`` is defined as ``-inf``);(NOTE: ``log`` without explicit;base is ``ln``, not ``log10``)	float[, float]	float
+``sin``;``cos``;``tan``;``cot``;``sec``;``csc``	trigonometric functions	float	float
+``arcsin``;``arccos``;``arctan``;``arccot``;``arcsec``;``arccsc``	inverse trigonometric functions	float	float
+``sinh``;``cosh``;``tanh``;``coth``;``sech``;``csch``	hyperbolic functions	float	float
+``arcsinh``;``arccosh``;``arctanh``;``arccoth``;``arcsech``;``arccsch``	inverse hyperbolic functions	float	float
+``piecewise(``;    ``true_value_1,``;      ``condition_1,``;    ``[true_value_2,``;      ``condition_2,]``;    ``[...]``;    ``[true_value_n,``;      ``condition_n,]``;    ``otherwise``;``)``	The function value is;the ``true_value*`` for the;first ``true`` ``condition*``;or ``otherwise`` if all;conditions are ``false``.	``*value*``: all float or all bool;``condition*``: all bool	float
+``abs(x)``	absolute value;``piecewise(x, x>=0, -x)``	float	float
+``sign(x)``	sign of ``x``;``piecewise(1, x>=0, -1)``	float	float
+``min(a, b)``;``max(a, b)``	minimum / maximum of {``a``, ``b``};``piecewise(a, a<=b, b)``;``piecewise(a, a>=b, b)``	float, float	float

doc/src/update_tables.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+
+import pandas as pd
+from pathlib import Path
+
+doc_dir = Path(__file__).parent.parent
+table_dir = Path(__file__).parent
+
+MULTILINE_DELIMITER = ";"
+tables = {
+    "Supported functions": {
+        "target": doc_dir / "documentation_data_format.rst",
+        "options": {
+            "header-rows": "1",
+            # "widths": "20 10 10 5",
+        },
+    },
+}
+
+
+def df_to_list_table(df, options, name):
+    columns = df.columns
+    table = f".. list-table:: {name}\n"
+    for option_id, option_value in options.items():
+        table += f"   :{option_id}: {option_value}\n"
+    table += "\n"
+
+    first = True
+    for column in columns:
+        if first:
+            table += "   * "
+            first = False
+        else:
+            table += "     "
+        table += f"- | {column}\n"
+
+    for _, row in df.iterrows():
+        first = True
+        for column in columns:
+            cell = row[column]
+            if first:
+                table += "   * "
+                first = False
+            else:
+                table += "     "
+            table += "- "
+            if MULTILINE_DELIMITER in cell:
+                first_line = True
+                for line in cell.split(MULTILINE_DELIMITER):
+                    if first_line:
+                        table += "| "
+                        first_line = False
+                    else:
+                        table += "       | "
+                    table += line
+                    table += "\n"
+            else:
+                table += cell
+                table += "\n"
+
+    return table
+
+
+def replace_text(filename, text, start, end):
+    with open(filename, "r") as f:
+        full_text0 = f.read()
+    before_start = full_text0.split(start)[0]
+    after_end = full_text0.split(end)[1]
+    full_text = (
+        before_start
+        + start
+        + text
+        + end
+        + after_end
+    )
+    with open(filename, "w") as f:
+        f.write(full_text)
+
+
+DISCLAIMER = "(GENERATED, DO NOT EDIT, INSTEAD EDIT IN PEtab/doc/src)"
+
+
+for table_id, table_data in tables.items():
+    target_file = table_data["target"]
+    options = table_data["options"]
+    df = pd.read_csv(table_dir/ f"{table_id}.tsv", sep="\t")
+    table = df_to_list_table(df, options=options, name=table_id)
+    replace_text(
+        filename=target_file,
+        text=table,
+        start=f"\n..\n   START TABLE {table_id} {DISCLAIMER}\n",
+        end=f"\n..\n   END TABLE {table_id}\n",
+    )