Merge pull request #12 from ESMValGroup/main

Update main to 2.8
malininae · Mar 29, 2023 · 66b9f3a · 66b9f3a
2 parents a8a5141 + 9dcaa55
commit 66b9f3a
Show file tree

Hide file tree

Showing 333 changed files with 34,693 additions and 6,109 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -2,9 +2,10 @@
 version: 2.1
 
 orbs:
-  coverage-reporter: codacy/coverage-reporter@11.9.2
+  coverage-reporter: codacy/coverage-reporter@13.13.0
 
 commands:
+
   check_changes:
     steps:
       - run:
@@ -19,20 +20,33 @@ commands:
               echo Skipping installation tests
               circleci step halt
             fi
+
+  log_versions:
+    steps:
+      - run:
+          name: Log versions
+          command: |
+            . /opt/conda/etc/profile.d/conda.sh
+            conda activate esmvaltool
+            esmvaltool version
+            dpkg -l > /logs/versions.txt
+            conda env export > /logs/environment.yml
+            pip freeze > /logs/requirements.txt
+
   test_and_report:
     steps:
       - run:
           name: Run tests
           command: |
             . /opt/conda/etc/profile.d/conda.sh
             conda activate esmvaltool
-            pytest -n 4 -m 'not flake8' --junitxml=test-reports/report.xml
+            pytest -n 4 --junitxml=test-reports/report.xml
             esmvaltool version
             esmvaltool -- --help
             ncl -V
             cdo --version
       - store_test_results:
-          path: test-reports/
+          path: test-reports/report.xml
       - store_artifacts:
           path: /logs
       - run:
@@ -47,6 +61,7 @@ commands:
           when: always
       - store_artifacts:
           path: test-reports.tar.gz
+
   test_installation_from_source:
     parameters:
       extra:
@@ -81,20 +96,13 @@ commands:
             then
               rm -r esmvaltool
             fi
-      - run:
-          name: Log versions
-          command: |
-            . /opt/conda/etc/profile.d/conda.sh
-            conda activate esmvaltool
-            dpkg -l > /logs/versions.txt
-            conda env export > /logs/environment.yml
-            pip freeze > /logs/requirements.txt
+      - log_versions
       - run:
           name: Lint source code
           command: |
             . /opt/conda/etc/profile.d/conda.sh
             conda activate esmvaltool
-            pytest -n 4 -m flake8
+            flake8 -j 4
       - test_and_report
       - save_cache:
           key: install-<< parameters.extra >>-{{ .Branch }}-{{ checksum "cache_key.txt" }}
@@ -120,15 +128,19 @@ jobs:
           name: Install dependencies
           command: |
             . /opt/conda/etc/profile.d/conda.sh
-            mkdir /logs
             conda activate esmvaltool
+            mkdir /logs
             pip install .[test] > /logs/install.txt 2>&1
-            # Run flake8 check with source code (it doesn't work without it)
-            pytest -n 4 -m flake8
-            # Update/install Julia dependencies
             esmvaltool install Julia > /logs/install_julia.txt 2>&1
-            # Remove source to test installed software
-            rm -r esmvaltool
+      - run:
+          name: Check Python code style and mistakes
+          command: |
+            . /opt/conda/etc/profile.d/conda.sh
+            conda activate esmvaltool
+            flake8 -j 4
+      - run:
+          name: Remove source code to test the installed software
+          command: rm -r esmvaltool
       - test_and_report
       - save_cache:
           key: test-{{ .Branch }}-{{ checksum "cache_key.txt" }}
@@ -143,7 +155,7 @@ jobs:
   test_installation_from_source_test_mode:
     # Test installation from source
     docker:
-      - image: condaforge/mambaforge
+      - image: condaforge/mambaforge:latest
     resource_class: large
     steps:
       - test_installation_from_source:
@@ -152,17 +164,74 @@ jobs:
   test_installation_from_source_develop_mode:
     # Test development installation
     docker:
-      - image: condaforge/mambaforge
+      - image: condaforge/mambaforge:latest
     resource_class: large
     steps:
       - test_installation_from_source:
           extra: develop
           flags: "--editable"
 
+  test_upstream_development:
+    # Test running recipes with the development version of ESMValCore. The
+    # purpose of this test to discover backward-incompatible changes early on in
+    # the development cycle.
+    docker:
+      - image: condaforge/mambaforge:latest
+    resource_class: large
+    steps:
+      - run:
+          name: Install git and ssh
+          environment:
+            DEBIAN_FRONTEND: noninteractive  # needed to install tzdata
+          command: apt update && apt install -y git ssh
+      - checkout
+      - run:
+          name: Generate cache key
+          command: echo $(date '+%Y')-$(expr $(date '+%V') / 2) | tee cache_key.txt
+      - restore_cache:
+          key: test-upstream-{{ .Branch }}-{{ checksum "cache_key.txt" }}
+      - run:
+          name: Install
+          command: |
+            # Install according to instructions on readthedocs with the
+            # development version of ESMValTool and ESMValCore:
+            # https://docs.esmvaltool.org/en/latest/quickstart/installation.html#install-from-source
+            . /opt/conda/etc/profile.d/conda.sh
+            mkdir /logs
+            mamba env create >> /logs/conda.txt 2>&1
+            conda activate esmvaltool
+            pip install --editable .[develop]
+            esmvaltool install Julia > /logs/install_julia.txt 2>&1
+            git clone https://github.com/ESMValGroup/ESMValCore $HOME/ESMValCore
+            pip install --editable $HOME/ESMValCore[develop]
+      - log_versions
+      - test_and_report
+      - run:
+          name: Run recipes
+          command: |
+            . /opt/conda/etc/profile.d/conda.sh
+            conda activate esmvaltool
+            mkdir -p ~/climate_data
+            esmvaltool config get_config_user
+            echo "offline: false" >> ~/.esmvaltool/config-user.yml
+            cat ~/.esmvaltool/config-user.yml
+            esmvaltool run examples/recipe_python.yml
+            for recipe in esmvaltool/recipes/testing/recipe_*.yml; do
+              esmvaltool run "$recipe"
+            done
+      - store_artifacts:
+          path: /root/esmvaltool_output
+      - save_cache:
+          key: test-upstream-{{ .Branch }}-{{ checksum "cache_key.txt" }}
+          paths:
+            - /opt/conda/pkgs
+            - /root/.cache/pip
+            - /root/climate_data
+
   build_documentation:
     # Test building documentation
     docker:
-      - image: condaforge/mambaforge
+      - image: condaforge/mambaforge:latest
     resource_class: small
     steps:
       - checkout
@@ -186,7 +255,7 @@ jobs:
   test_installation_from_conda:
     # Test conda package installation
     docker:
-      - image: condaforge/mambaforge
+      - image: condaforge/mambaforge:latest
     resource_class: medium
     steps:
       - run:
@@ -229,5 +298,6 @@ workflows:
       - run_tests
       - test_installation_from_source_test_mode
       - test_installation_from_source_develop_mode
+      - test_upstream_development
       - build_documentation
       - test_installation_from_conda
diff --git a/.github/workflows/create-condalock-file.yml b/.github/workflows/create-condalock-file.yml
@@ -63,6 +63,8 @@ jobs:
         run: esmvaltool --help
       - shell: bash -l {0}
         run: esmvaltool version
+      - shell: bash -l {0}
+        run: flake8
       - shell: bash -l {0}
         run: pytest -n 2 -m "not installation"
       # Automated PR

diff --git a/.github/workflows/install-from-condalock-file.yml b/.github/workflows/install-from-condalock-file.yml
@@ -11,7 +11,7 @@ on:
   push:
     branches:
       - main
-  #     - condalock-update
+  #    - condalock-update
   # run the test only if the PR is to main
   # turn it on if required
   #pull_request:
@@ -58,6 +58,8 @@ jobs:
         run: esmvaltool --help
       - shell: bash -l {0}
         run: esmvaltool version 2>&1 | tee source_install_linux_artifacts_python_${{ matrix.python-version }}/version.txt
+      - shell: bash -l {0}
+        run: flake8
       - shell: bash -l {0}
         run: pytest -n 2 -m "not installation"
       - name: Upload artifacts

diff --git a/.github/workflows/test-development.yml b/.github/workflows/test-development.yml
@@ -55,6 +55,9 @@ jobs:
           git clone https://github.com/ESMValGroup/ESMValCore.git
           cd ESMValCore
           pip install -e .[develop]
+      - name: Run flake8
+        shell: bash -l {0}
+        run: flake8
       - name: Run tests
         shell: bash -l {0}
         run: pytest -n 2 -m "not installation" 2>&1 | tee develop_test_linux_artifacts_python_${{ matrix.python-version }}/test_report.txt

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -42,6 +42,9 @@ jobs:
       - name: Install Julia dependencies
         shell: bash -l {0}
         run: esmvaltool install Julia
+      - name: Run flake8
+        shell: bash -l {0}
+        run: flake8
       - name: Run tests
         shell: bash -l {0}
         run: pytest -n 2 -m "not installation" 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/test_report.txt
@@ -83,6 +86,9 @@ jobs:
       - name: Install ESMValTool
         shell: bash -l {0}
         run: pip install -e .[develop] 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/install.txt
+      - name: Run flake8
+        shell: bash -l {0}
+        run: flake8
       - name: Run tests
         shell: bash -l {0}
         run: pytest -n 2 -m "not installation" 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/test_report.txt

diff --git a/.readthedocs.yaml b/.readthedocs.yaml
@@ -10,19 +10,15 @@ build:
   os: ubuntu-22.04
   tools:
     python: "mambaforge-4.10"
+  jobs:
+    post_create_environment:
+      - pip install . --no-deps
 
 # Declare the requirements required to build your docs
 conda:
   environment:
     environment.yml
 
-python:
-  install:
-    - method: pip
-      path: .
-      extra_requirements:
-        - doc
-
 # Build documentation in the doc directory with Sphinx
 sphinx:
   configuration: doc/sphinx/source/conf.py

diff --git a/CITATION.cff b/CITATION.cff
@@ -340,13 +340,22 @@ authors:
     family-names: Kazeroni
     given-names: Rémi
     orcid: "https://orcid.org/0000-0001-7205-9528"
+  -
+    affiliation: "NASA, USA"
+    family-names: Potter
+    given-names: Jerry
+  -
+    affiliation: "DLR, Germany"
+    family-names: Winterstein
+    given-names: Franziska
+    orcid: "https://orcid.org/0000-0002-2406-4936"
 
 cff-version: 1.2.0
-date-released: 2022-07-25
+date-released: 2023-03-28
 doi: "10.5281/zenodo.3401363"
 license: "Apache-2.0"
 message: "If you use this software, please cite it using these metadata."
 repository-code: "https://github.com/ESMValGroup/ESMValTool/"
 title: ESMValTool
-version: "v2.6.0"
+version: "v2.8.0"
 ...
diff --git a/README.md b/README.md
@@ -8,7 +8,7 @@
 [![Codacy Badge](https://app.codacy.com/project/badge/Coverage/79bf6932c2e844eea15d0fb1ed7e415c)](https://www.codacy.com/gh/ESMValGroup/ESMValTool?utm_source=github.com&utm_medium=referral&utm_content=ESMValGroup/ESMValTool&utm_campaign=Badge_Coverage)
 [![Codacy Badge](https://app.codacy.com/project/badge/Grade/79bf6932c2e844eea15d0fb1ed7e415c)](https://www.codacy.com/gh/ESMValGroup/ESMValTool?utm_source=github.com&amp;utm_medium=referral&amp;utm_content=ESMValGroup/ESMValTool&amp;utm_campaign=Badge_Grade)
 [![Docker Build Status](https://img.shields.io/docker/cloud/build/esmvalgroup/esmvaltool.svg)](https://hub.docker.com/r/esmvalgroup/esmvaltool/)
-[![Anaconda-Server Badge](https://anaconda.org/esmvalgroup/esmvaltool/badges/installer/conda.svg)](https://conda.anaconda.org/esmvalgroup)
+[![Anaconda-Server Badge](https://anaconda.org/conda-forge/esmvaltool/badges/version.svg)](https://anaconda.org/conda-forge/esmvaltool)
 ![stand with Ukraine](https://badgen.net/badge/stand%20with/UKRAINE/?color=0057B8&labelColor=FFD700)
 
 ![esmvaltoollogo](https://github.com/ESMValGroup/ESMValTool/blob/main/doc/sphinx/source/figures/ESMValTool-logo-2.png)