Build the MUSICA library in CAM-SIMA

.dockerignore

@@ -1,14 +1,11 @@
-# ignore all
-*
+.github/
+.gitignore
+LICENSE
+README.md
 
-# include things to copy
-!Externals*
-!src/
-!cime_config/
-!manage_externals/
-!test/
-!.config_files.xml
-!docker
-!bin/
-!.lib/
-!.gitmodules
+# Ignore editor temporaries and backups
+*.swp
+*~
+.#*
+\#*#
+**/.vscode/

README.md

@@ -16,7 +16,7 @@ cd CAM-SIMA
 ## NOTE: This is **unsupported** development code and is subject to the [CESM developer's agreement](http://www.cgd.ucar.edu/cseg/development-code.html).
 ```
 git checkout development
-./manage_externals/checkout_externals
+./bin/git-fleximod update
 ```
 
 Good luck, and have a great day!

cime_config/atm_musica_config.py

@@ -0,0 +1,9 @@
+"""
+The URLs and tags provided in this script are read by buildlib to build
+and install the MUSICA library, as well as to download the MUSICA configuration."
+"""
+
+MUSICA_REPO_URL = "https://github.com/NCAR/musica.git"
+MUSICA_TAG = "802bc9041f964a7edd5ec896bec8cdc0c90de50b"
+CHEMISTRY_DATA_REPO_URL = "https://github.com/NCAR/cam-sima-chemistry-data.git"
+CHEMISTRY_DATA_TAG = "b81cbc2e61c41ecd2d09167d6736daf1bf1be149"

cime_config/buildlib

@@ -8,9 +8,12 @@ import sys
 import os
 import filecmp
 import shutil
+import subprocess
 import logging
 
 from cam_config import ConfigCAM # CAM's configure structure
+from atm_musica_config import MUSICA_REPO_URL, MUSICA_TAG
+from atm_musica_config import CHEMISTRY_DATA_REPO_URL, CHEMISTRY_DATA_TAG
 
 # Check for the CIME library, and add it
 # to the python path:
@@ -22,7 +25,7 @@ sys.path.append(os.path.join(__CIMEROOT, "CIME", "Tools"))
 #pylint: disable=wrong-import-position
 # CIME imports
 from CIME.case import Case
-from CIME.utils import run_cmd, expect
+from CIME.utils import run_cmd, expect, symlink_force
 from CIME.utils import stop_buffering_output
 from CIME.buildlib import parse_input
 from CIME.build import get_standard_makefile_args
@@ -76,6 +79,7 @@ def _build_cam():
         config.generate_cam_src(gen_indent)
 
         dycore = config.get_value('dyn')
+        phys_suites = config.get_value('physics_suites')
         reg_dir = config.get_value('reg_dir')
         init_dir = config.get_value('init_dir')
         phys_dirs_str = config.get_value('phys_dirs')
@@ -122,12 +126,6 @@ def _build_cam():
             paths.append(os.path.join(atm_root, "src", "dynamics", "tests",
                                       "initial_conditions"))
 
-        # If using the CMEPS/NUOPC coupler, then add additional path:
-        if case.get_value("COMP_INTERFACE") == "nuopc":
-            paths.append(os.path.join(__CIMEROOT, "src", "drivers",
-                                      "nuopc", "nuopc_cap_share"))
-        # End if
-
         # Write Filepath text file
         with open(filepath_src, "w", encoding='utf-8') as filepath:
             filepath.write("\n".join(paths))
@@ -174,11 +172,26 @@ def _build_cam():
             if len(optional_mpas_features) > 0:
                 cmd += " OPTIONAL_MPAS_FEATURES=\"" + " ".join(optional_mpas_features) + "\""
 
+        # If the physics suite is MUSICA, build the MUSICA library and get configuration
+        if phys_suites == "musica":
+            _build_musica_library(case)
+            _download_musica_configuration(caseroot)
+
+            cmd += ' USER_INCLDIR="'\
+                    f'-I{os.path.join(bldroot, "musica", "include", "micm")} '\
+                    f'-I{os.path.join(bldroot, "musica", "include", "musica")} '\
+                    f'-I{os.path.join(bldroot, "musica", "include", "musica", "micm")} '\
+                    f'-I{os.path.join(bldroot, "musica", "include", "musica", "tuvx")} '\
+                    f'-I{os.path.join(bldroot, "musica", "include", "musica", "fortran")} '\
+                    '"'
+
         retcode, out, err = run_cmd(cmd)
         _LOGGER.info("Command %s:\n\nstdout:\n%s\n\nstderr:\n%s\n", cmd, out, err)
         expect(retcode == 0, f"Command {cmd} failed with rc={retcode}")
 
+###############################################################################
 def _prepare_mpas(case: Case) -> None:
+###############################################################################
     """
     Prepare MPAS build infrastructure.
     """
@@ -206,7 +219,9 @@ def _prepare_mpas(case: Case) -> None:
     shutil.copytree(os.path.normpath(os.path.join(mpas_dycore_src_root, os.pardir, os.pardir, "assets")), mpas_dycore_bld_root, copy_function=_copy2_as_needed, dirs_exist_ok=True)
     shutil.copytree(os.path.normpath(os.path.join(mpas_dycore_src_root, os.pardir, os.pardir, "driver")), os.path.join(mpas_dycore_bld_root, "driver"), copy_function=_copy2_as_needed, dirs_exist_ok=True)
 
+###############################################################################
 def _copy2_as_needed(src: str, dst: str) -> None:
+###############################################################################
     """
     Wrapper around `shutil.copy2`. Copy the file `src` to the file or directory `dst` as needed.
     """
@@ -226,6 +241,147 @@ def _copy2_as_needed(src: str, dst: str) -> None:
                 # Example scenario: User added some new source code files.
                 shutil.copy2(src, dst)
 
+###############################################################################
+def _build_musica_library(case: Case) -> None:
+###############################################################################
+    """
+    Builds and installs the MUSICA library.
+
+    Args:
+        case (Case)
+
+    Raises:
+        Exception: If configuring the CMake MUSICA project fails or
+            the MUSICA library build fails, an exception is raised.
+
+    """
+    install_dir_name = "install"
+    build_type = "Release"
+    caseroot = os.path.normpath(case.get_value("CASEROOT"))
+    _clone_and_checkout(MUSICA_REPO_URL, MUSICA_TAG, caseroot)
+
+    bld_path = os.path.join(caseroot, "musica", "build")
+    if os.path.exists(bld_path):
+        shutil.rmtree(bld_path)
+    os.makedirs(bld_path)
+
+    # To install the target, the working directory must be the build directory.
+    current_dir = os.getcwd()
+    os.chdir(bld_path)
+
+    command = [
+        "cmake",
+        f"-DCMAKE_INSTALL_PREFIX={install_dir_name}",
+        f"-DCMAKE_BUILD_TYPE={build_type}",
+        "-DMUSICA_ENABLE_TESTS=OFF",
+        "-DMUSICA_BUILD_FORTRAN_INTERFACE=ON",
+        ".."
+    ]
+    result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=False)
+    if result.returncode != 0:
+        raise Exception(f"Unable to configure the MUSICA CMake project. Error: {result.stderr}")
+
+    command = ["cmake", "--build", ".", "--target", "install"]
+    result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=False)
+    if result.returncode != 0:
+        raise Exception(f"Unable to build the MUSICA library. Error: {result.stderr}")
+
+    # Create symlinks pointing to the MUSICA libraries in the ATM build directory
+    atm_bld_root = os.path.join(case.get_value("EXEROOT"), "atm", "obj")
+    musica_lib_root = os.path.join(atm_bld_root, "musica", "lib")
+    os.makedirs(musica_lib_root, exist_ok=True)
+
+    installed_lib_path = os.path.join(bld_path, install_dir_name, "lib64")
+    root, _, files = next(os.walk(installed_lib_path, topdown=True))
+    for file in files:
+        symlink_force(os.path.join(root, file), os.path.join(musica_lib_root, file))
+
+    # Create symlinks pointing to the MUSICA include files in the ATM build directory
+    musica_include_root = os.path.join(atm_bld_root, "musica", "include")
+    os.makedirs(musica_include_root, exist_ok=True)
+
+    installed_include_path = os.path.join(bld_path, install_dir_name, "include")
+    for root, _, files in os.walk(installed_include_path):
+        rel_path = os.path.relpath(root, installed_include_path)
+        dest_dir = os.path.join(musica_include_root, rel_path)
+
+        os.makedirs(dest_dir, exist_ok=True)
+        for file in files:
+            symlink_force(os.path.join(root, file), os.path.join(dest_dir, file))
+
+    os.chdir(current_dir)
+
+###############################################################################
+def _download_musica_configuration(download_dest: str) -> None:
+###############################################################################
+    """
+    Downloads the MUSICA configuration and renames the configuration
+    directory to match the name in the MUSICA-CCPP configuration.
+
+    Args:
+        download_dest: destination where configuration will be downloaded.
+
+    Raises:
+        Exception: If the directory to be renamed is not found or
+            any other exceptions occur during the renaming process,
+            an exception is raised with the error message.
+    """
+    musica_config_dir_name = "musica_configurations"
+
+    _clone_and_checkout(CHEMISTRY_DATA_REPO_URL, CHEMISTRY_DATA_TAG, download_dest)
+
+    original_dir = os.path.join(download_dest, "cam-sima-chemistry-data", "mechanisms")
+    renamed_dir = os.path.join(download_dest, "cam-sima-chemistry-data", musica_config_dir_name)
+    try:
+        os.rename(original_dir, renamed_dir)
+    except FileNotFoundError:
+        raise FileNotFoundError(f"The directory '{original_dir}' was not found.")
+    except FileExistsError:
+        raise FileExistsError(f"The destination directory '{renamed_dir}' already exists.")
+    except PermissionError:
+        raise PermissionError(f"Permission denied to rename '{original_dir}'.")
+    except OSError as e:
+        raise OSError(f"An error occurred while renaming: {e}")
+
+    musica_config_path = os.path.join(download_dest, musica_config_dir_name)
+    if os.path.exists(musica_config_path):
+        shutil.rmtree(musica_config_path)
+
+    shutil.move(renamed_dir, download_dest)
+
+###############################################################################
+def _clone_and_checkout(repo_url: str, tag_name: str, clone_dest: str):
+###############################################################################
+    """
+    Clones a Git repository from the URL and checks out a specific branch.
+
+    Args:
+        repo_url: The URL of the Git repository to clone
+        tag_name: The tag name to check out
+        clone_dest: destination where the repository will be cloned.
+
+    Raises:
+        Exception: If the `git clone` or `git checkout` commands fail,
+            an exception is raised with the error message.
+    """
+    repo_name = repo_url.split("/")[-1].replace(".git", "")
+    repo_path = os.path.join(clone_dest, repo_name)
+
+    if os.path.exists(repo_path):
+        shutil.rmtree(repo_path)
+
+    result = subprocess.run(["git", "clone", repo_url, repo_path],
+                            stderr=subprocess.PIPE, text=True, check=False)
+    if result.returncode != 0:
+        raise Exception(f"Unable to clone the repository: {repo_url}. \
+                        Error: {result.stderr}")
+
+    result = subprocess.run(["git", "-C", repo_path, "checkout", tag_name],
+                            stderr=subprocess.PIPE, text=True, check=False)
+    if result.returncode != 0:
+        raise Exception(f"Unable to checkout the branch: {tag_name}. \
+                        Error: {result.stderr}")
+
 ###############################################################################
 
 if __name__ == "__main__":

docker/Dockerfile

@@ -19,7 +19,8 @@ RUN dnf -y update \
       vim \
     && dnf clean all
 
-RUN ln -s $(which python3) /usr/bin/python && \
+RUN rm -f /usr/bin/python && \
+    ln -s $(which python3) /usr/bin/python && \
     pip install --upgrade pip && \
     pip install --upgrade setuptools
 

docker/Dockerfile.esmf

@@ -29,7 +29,7 @@ ENV OMP_NUM_THREADS=5
 ## Build and install ESMF
 ###################################################
 
-ENV ESMF_TAG="8.4.2"
+ENV ESMF_TAG="8.6.0"
 
 # set necessary environment variables
 ENV ESMF_DIR=/esmf-${ESMF_TAG}
@@ -51,4 +51,4 @@ RUN wget -q https://github.com/esmf-org/esmf/archive/refs/tags/v${ESMF_TAG}.tar.
     # This command lets you see what esmf thinks its build options are but may not necessary to build, not sure
     make info && \
     make -j 8 && \
-    make install
+    make install

docker/Dockerfile.musica

@@ -2,6 +2,8 @@
 # esmf is am image you are expected to have built. Read the README file for instructions
 FROM --platform=linux/amd64 esmf:latest
 
+ARG BUILD_TYPE=Debug
+
 ###################################################
 ## Install necessary packages
 ###################################################
@@ -19,7 +21,8 @@ RUN dnf -y update \
       vim \
     && dnf clean all
 
-RUN ln -s $(which python3) /usr/bin/python && \
+RUN rm -f /usr/bin/python && \
+    ln -s $(which python3) /usr/bin/python && \
     pip install --upgrade pip && \
     pip install --upgrade setuptools
 
@@ -39,71 +42,48 @@ RUN cd pnetcdf-1.12.3 && \
      make -j 8 install && \
      ldconfig
 
-ENV FC=gfortran
-
-###################################################
-## Build and install MUSICA
-###################################################
-
-RUN git clone https://github.com/NCAR/musica.git \
-    && cd musica \
-    && git checkout 2a5eeaac982a3eb80b96d1e2087b91b301d1e748
-
-RUN mkdir /musica/build \
-    && cd /musica/build \
-    && cmake \
-             -D ENABLE_TESTS=OFF \
-             -D MUSICA_BUILD_FORTRAN_INTERFACE=ON \
-             .. \
-    && make install -j 8
-
 ###################################################
 ## Build CAM-SIMA
 ###################################################
 
-# create a user to run the case
+# Create a user to run the case
 RUN adduser cam_sima_user \
     && echo "cam_sima_user ALL=(root) NOPASSWD:ALL" > /etc/sudoers.d/cam_sima_user \
     && chmod 0440 /etc/sudoers.d/cam_sima_user
 
-# copy in the CAM-SIMA code and give the proper user permissions
+# Copy in the CAM-SIMA code and give the proper user permissions
 COPY --chown=cam_sima_user . /home/cam_sima_user/CAM-SIMA
 
 USER cam_sima_user
 WORKDIR /home/cam_sima_user/CAM-SIMA
 
-# pull the dependencies
+# Pull the dependencies
 RUN ./bin/git-fleximod update
 
 # Copy in the machine information for the container
 RUN cp /home/cam_sima_user/CAM-SIMA/docker/config_machines.xml /home/cam_sima_user/CAM-SIMA/ccs_config/machines/
 
 # Set environment variables needed to create and build the case
 ENV USER=$(whoami)
-ENV CASE_NAME=/home/cam_sima_user/case_name
+ENV CASE_NAME=/home/cam_sima_user/case_name/test-case
 ENV CESMDATAROOT=/home/cam_sima_user/cesm_data
 ENV CIME_MACHINE=container
 ENV CIME_MODEL=cesm
 ENV ESMFMKFILE=/usr/local/lib/esmf.mk
 
 # Create a case
-RUN ./cime/scripts/create_newcase --case $CASE_NAME  --compset FPHYStest --res ne5_ne5_mg37 --run-unsupported
+RUN /home/cam_sima_user/CAM-SIMA/cime/scripts/create_newcase --case $CASE_NAME \
+    --compset FPHYStest --res ne5_ne5_mg37 --run-unsupported
 
 WORKDIR $CASE_NAME 
 
 RUN ./case.setup
 
 RUN ./xmlchange CAM_CONFIG_OPTS="--dyn none --physics-suites musica"
 RUN ./xmlchange CAM_LINKED_LIBS="-lmusica-fortran -lmusica -lyaml-cpp"
+RUN ./xmlchange CAM_LINKED_LIBS="-L/home/cam_sima_user/scratch/test-case/bld/atm/obj/musica/lib -lmusica-fortran -lmusica -lyaml-cpp"
 RUN ./xmlchange ROF_NCPL=48
 RUN ./xmlchange STOP_OPTION=nsteps
 RUN ./xmlchange STOP_N=5
 
-# Copy in the grid files and a snapshot file
-RUN chmod +x /home/cam_sima_user/CAM-SIMA/docker/ftp_download.sh
-RUN /home/cam_sima_user/CAM-SIMA/docker/ftp_download.sh
-
-# # add the snapshot file
-RUN echo "ncdata='/home/cam_sima_user/run_heldsuarez_cam6_nt2_bigg_try005.cam.h5.0001-01-01-00000.nc'" >> user_nl_cam
-
 RUN ./case.build