diff --git a/.dockerignore b/.dockerignore index 12b85fe8..45066338 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,14 +1,11 @@ -# ignore all -* +.github/ +.gitignore +LICENSE +README.md -# include things to copy -!Externals* -!src/ -!cime_config/ -!manage_externals/ -!test/ -!.config_files.xml -!docker -!bin/ -!.lib/ -!.gitmodules +# Ignore editor temporaries and backups +*.swp +*~ +.#* +\#*# +**/.vscode/ \ No newline at end of file diff --git a/README.md b/README.md index 35b3bcbb..6358f935 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ cd CAM-SIMA ## NOTE: This is **unsupported** development code and is subject to the [CESM developer's agreement](http://www.cgd.ucar.edu/cseg/development-code.html). ``` git checkout development -./manage_externals/checkout_externals +./bin/git-fleximod update ``` Good luck, and have a great day! diff --git a/cime_config/atm_musica_config.py b/cime_config/atm_musica_config.py new file mode 100644 index 00000000..6bbf1b5c --- /dev/null +++ b/cime_config/atm_musica_config.py @@ -0,0 +1,9 @@ +""" +The URLs and tags provided in this script are read by buildlib to build +and install the MUSICA library, as well as to download the MUSICA configuration." +""" + +MUSICA_REPO_URL = "https://github.com/NCAR/musica.git" +MUSICA_TAG = "802bc9041f964a7edd5ec896bec8cdc0c90de50b" +CHEMISTRY_DATA_REPO_URL = "https://github.com/NCAR/cam-sima-chemistry-data.git" +CHEMISTRY_DATA_TAG = "b81cbc2e61c41ecd2d09167d6736daf1bf1be149" diff --git a/cime_config/buildlib b/cime_config/buildlib index 7c8b629e..4fd0c517 100755 --- a/cime_config/buildlib +++ b/cime_config/buildlib @@ -8,9 +8,12 @@ import sys import os import filecmp import shutil +import subprocess import logging from cam_config import ConfigCAM # CAM's configure structure +from atm_musica_config import MUSICA_REPO_URL, MUSICA_TAG +from atm_musica_config import CHEMISTRY_DATA_REPO_URL, CHEMISTRY_DATA_TAG # Check for the CIME library, and add it # to the python path: @@ -22,7 +25,7 @@ sys.path.append(os.path.join(__CIMEROOT, "CIME", "Tools")) #pylint: disable=wrong-import-position # CIME imports from CIME.case import Case -from CIME.utils import run_cmd, expect +from CIME.utils import run_cmd, expect, symlink_force from CIME.utils import stop_buffering_output from CIME.buildlib import parse_input from CIME.build import get_standard_makefile_args @@ -76,6 +79,7 @@ def _build_cam(): config.generate_cam_src(gen_indent) dycore = config.get_value('dyn') + phys_suites = config.get_value('physics_suites') reg_dir = config.get_value('reg_dir') init_dir = config.get_value('init_dir') phys_dirs_str = config.get_value('phys_dirs') @@ -122,12 +126,6 @@ def _build_cam(): paths.append(os.path.join(atm_root, "src", "dynamics", "tests", "initial_conditions")) - # If using the CMEPS/NUOPC coupler, then add additional path: - if case.get_value("COMP_INTERFACE") == "nuopc": - paths.append(os.path.join(__CIMEROOT, "src", "drivers", - "nuopc", "nuopc_cap_share")) - # End if - # Write Filepath text file with open(filepath_src, "w", encoding='utf-8') as filepath: filepath.write("\n".join(paths)) @@ -174,11 +172,26 @@ def _build_cam(): if len(optional_mpas_features) > 0: cmd += " OPTIONAL_MPAS_FEATURES=\"" + " ".join(optional_mpas_features) + "\"" + # If the physics suite is MUSICA, build the MUSICA library and get configuration + if phys_suites == "musica": + _build_musica_library(case) + _download_musica_configuration(caseroot) + + cmd += ' USER_INCLDIR="'\ + f'-I{os.path.join(bldroot, "musica", "include", "micm")} '\ + f'-I{os.path.join(bldroot, "musica", "include", "musica")} '\ + f'-I{os.path.join(bldroot, "musica", "include", "musica", "micm")} '\ + f'-I{os.path.join(bldroot, "musica", "include", "musica", "tuvx")} '\ + f'-I{os.path.join(bldroot, "musica", "include", "musica", "fortran")} '\ + '"' + retcode, out, err = run_cmd(cmd) _LOGGER.info("Command %s:\n\nstdout:\n%s\n\nstderr:\n%s\n", cmd, out, err) expect(retcode == 0, f"Command {cmd} failed with rc={retcode}") +############################################################################### def _prepare_mpas(case: Case) -> None: +############################################################################### """ Prepare MPAS build infrastructure. """ @@ -206,7 +219,9 @@ def _prepare_mpas(case: Case) -> None: shutil.copytree(os.path.normpath(os.path.join(mpas_dycore_src_root, os.pardir, os.pardir, "assets")), mpas_dycore_bld_root, copy_function=_copy2_as_needed, dirs_exist_ok=True) shutil.copytree(os.path.normpath(os.path.join(mpas_dycore_src_root, os.pardir, os.pardir, "driver")), os.path.join(mpas_dycore_bld_root, "driver"), copy_function=_copy2_as_needed, dirs_exist_ok=True) +############################################################################### def _copy2_as_needed(src: str, dst: str) -> None: +############################################################################### """ Wrapper around `shutil.copy2`. Copy the file `src` to the file or directory `dst` as needed. """ @@ -226,6 +241,147 @@ def _copy2_as_needed(src: str, dst: str) -> None: # Example scenario: User added some new source code files. shutil.copy2(src, dst) +############################################################################### +def _build_musica_library(case: Case) -> None: +############################################################################### + """ + Builds and installs the MUSICA library. + + Args: + case (Case) + + Raises: + Exception: If configuring the CMake MUSICA project fails or + the MUSICA library build fails, an exception is raised. + + """ + install_dir_name = "install" + build_type = "Release" + caseroot = os.path.normpath(case.get_value("CASEROOT")) + _clone_and_checkout(MUSICA_REPO_URL, MUSICA_TAG, caseroot) + + bld_path = os.path.join(caseroot, "musica", "build") + if os.path.exists(bld_path): + shutil.rmtree(bld_path) + os.makedirs(bld_path) + + # To install the target, the working directory must be the build directory. + current_dir = os.getcwd() + os.chdir(bld_path) + + command = [ + "cmake", + f"-DCMAKE_INSTALL_PREFIX={install_dir_name}", + f"-DCMAKE_BUILD_TYPE={build_type}", + "-DMUSICA_ENABLE_TESTS=OFF", + "-DMUSICA_BUILD_FORTRAN_INTERFACE=ON", + ".." + ] + result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=False) + if result.returncode != 0: + raise Exception(f"Unable to configure the MUSICA CMake project. Error: {result.stderr}") + + command = ["cmake", "--build", ".", "--target", "install"] + result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=False) + if result.returncode != 0: + raise Exception(f"Unable to build the MUSICA library. Error: {result.stderr}") + + # Create symlinks pointing to the MUSICA libraries in the ATM build directory + atm_bld_root = os.path.join(case.get_value("EXEROOT"), "atm", "obj") + musica_lib_root = os.path.join(atm_bld_root, "musica", "lib") + os.makedirs(musica_lib_root, exist_ok=True) + + installed_lib_path = os.path.join(bld_path, install_dir_name, "lib64") + root, _, files = next(os.walk(installed_lib_path, topdown=True)) + for file in files: + symlink_force(os.path.join(root, file), os.path.join(musica_lib_root, file)) + + # Create symlinks pointing to the MUSICA include files in the ATM build directory + musica_include_root = os.path.join(atm_bld_root, "musica", "include") + os.makedirs(musica_include_root, exist_ok=True) + + installed_include_path = os.path.join(bld_path, install_dir_name, "include") + for root, _, files in os.walk(installed_include_path): + rel_path = os.path.relpath(root, installed_include_path) + dest_dir = os.path.join(musica_include_root, rel_path) + + os.makedirs(dest_dir, exist_ok=True) + for file in files: + symlink_force(os.path.join(root, file), os.path.join(dest_dir, file)) + + os.chdir(current_dir) + +############################################################################### +def _download_musica_configuration(download_dest: str) -> None: +############################################################################### + """ + Downloads the MUSICA configuration and renames the configuration + directory to match the name in the MUSICA-CCPP configuration. + + Args: + download_dest: destination where configuration will be downloaded. + + Raises: + Exception: If the directory to be renamed is not found or + any other exceptions occur during the renaming process, + an exception is raised with the error message. + """ + musica_config_dir_name = "musica_configurations" + + _clone_and_checkout(CHEMISTRY_DATA_REPO_URL, CHEMISTRY_DATA_TAG, download_dest) + + original_dir = os.path.join(download_dest, "cam-sima-chemistry-data", "mechanisms") + renamed_dir = os.path.join(download_dest, "cam-sima-chemistry-data", musica_config_dir_name) + try: + os.rename(original_dir, renamed_dir) + except FileNotFoundError: + raise FileNotFoundError(f"The directory '{original_dir}' was not found.") + except FileExistsError: + raise FileExistsError(f"The destination directory '{renamed_dir}' already exists.") + except PermissionError: + raise PermissionError(f"Permission denied to rename '{original_dir}'.") + except OSError as e: + raise OSError(f"An error occurred while renaming: {e}") + + musica_config_path = os.path.join(download_dest, musica_config_dir_name) + if os.path.exists(musica_config_path): + shutil.rmtree(musica_config_path) + + shutil.move(renamed_dir, download_dest) + +############################################################################### +def _clone_and_checkout(repo_url: str, tag_name: str, clone_dest: str): +############################################################################### + """ + Clones a Git repository from the URL and checks out a specific branch. + + Args: + repo_url: The URL of the Git repository to clone + tag_name: The tag name to check out + clone_dest: destination where the repository will be cloned. + + Raises: + Exception: If the `git clone` or `git checkout` commands fail, + an exception is raised with the error message. + """ + repo_name = repo_url.split("/")[-1].replace(".git", "") + repo_path = os.path.join(clone_dest, repo_name) + + if os.path.exists(repo_path): + shutil.rmtree(repo_path) + + result = subprocess.run(["git", "clone", repo_url, repo_path], + stderr=subprocess.PIPE, text=True, check=False) + if result.returncode != 0: + raise Exception(f"Unable to clone the repository: {repo_url}. \ + Error: {result.stderr}") + + result = subprocess.run(["git", "-C", repo_path, "checkout", tag_name], + stderr=subprocess.PIPE, text=True, check=False) + if result.returncode != 0: + raise Exception(f"Unable to checkout the branch: {tag_name}. \ + Error: {result.stderr}") + ############################################################################### if __name__ == "__main__": diff --git a/docker/Dockerfile b/docker/Dockerfile index 098db37d..5e3e33e5 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -19,7 +19,8 @@ RUN dnf -y update \ vim \ && dnf clean all -RUN ln -s $(which python3) /usr/bin/python && \ +RUN rm -f /usr/bin/python && \ + ln -s $(which python3) /usr/bin/python && \ pip install --upgrade pip && \ pip install --upgrade setuptools diff --git a/docker/Dockerfile.esmf b/docker/Dockerfile.esmf index 4a678b32..ff6976b2 100644 --- a/docker/Dockerfile.esmf +++ b/docker/Dockerfile.esmf @@ -29,7 +29,7 @@ ENV OMP_NUM_THREADS=5 ## Build and install ESMF ################################################### -ENV ESMF_TAG="8.4.2" +ENV ESMF_TAG="8.6.0" # set necessary environment variables ENV ESMF_DIR=/esmf-${ESMF_TAG} @@ -51,4 +51,4 @@ RUN wget -q https://github.com/esmf-org/esmf/archive/refs/tags/v${ESMF_TAG}.tar. # This command lets you see what esmf thinks its build options are but may not necessary to build, not sure make info && \ make -j 8 && \ - make install + make install \ No newline at end of file diff --git a/docker/Dockerfile.musica b/docker/Dockerfile.musica index 0f59f21d..810927e2 100644 --- a/docker/Dockerfile.musica +++ b/docker/Dockerfile.musica @@ -2,6 +2,8 @@ # esmf is am image you are expected to have built. Read the README file for instructions FROM --platform=linux/amd64 esmf:latest +ARG BUILD_TYPE=Debug + ################################################### ## Install necessary packages ################################################### @@ -19,7 +21,8 @@ RUN dnf -y update \ vim \ && dnf clean all -RUN ln -s $(which python3) /usr/bin/python && \ +RUN rm -f /usr/bin/python && \ + ln -s $(which python3) /usr/bin/python && \ pip install --upgrade pip && \ pip install --upgrade setuptools @@ -39,40 +42,22 @@ RUN cd pnetcdf-1.12.3 && \ make -j 8 install && \ ldconfig -ENV FC=gfortran - -################################################### -## Build and install MUSICA -################################################### - -RUN git clone https://github.com/NCAR/musica.git \ - && cd musica \ - && git checkout 2a5eeaac982a3eb80b96d1e2087b91b301d1e748 - -RUN mkdir /musica/build \ - && cd /musica/build \ - && cmake \ - -D ENABLE_TESTS=OFF \ - -D MUSICA_BUILD_FORTRAN_INTERFACE=ON \ - .. \ - && make install -j 8 - ################################################### ## Build CAM-SIMA ################################################### -# create a user to run the case +# Create a user to run the case RUN adduser cam_sima_user \ && echo "cam_sima_user ALL=(root) NOPASSWD:ALL" > /etc/sudoers.d/cam_sima_user \ && chmod 0440 /etc/sudoers.d/cam_sima_user -# copy in the CAM-SIMA code and give the proper user permissions +# Copy in the CAM-SIMA code and give the proper user permissions COPY --chown=cam_sima_user . /home/cam_sima_user/CAM-SIMA USER cam_sima_user WORKDIR /home/cam_sima_user/CAM-SIMA -# pull the dependencies +# Pull the dependencies RUN ./bin/git-fleximod update # Copy in the machine information for the container @@ -80,14 +65,15 @@ RUN cp /home/cam_sima_user/CAM-SIMA/docker/config_machines.xml /home/cam_sima_us # Set environment variables needed to create and build the case ENV USER=$(whoami) -ENV CASE_NAME=/home/cam_sima_user/case_name +ENV CASE_NAME=/home/cam_sima_user/case_name/test-case ENV CESMDATAROOT=/home/cam_sima_user/cesm_data ENV CIME_MACHINE=container ENV CIME_MODEL=cesm ENV ESMFMKFILE=/usr/local/lib/esmf.mk # Create a case -RUN ./cime/scripts/create_newcase --case $CASE_NAME --compset FPHYStest --res ne5_ne5_mg37 --run-unsupported +RUN /home/cam_sima_user/CAM-SIMA/cime/scripts/create_newcase --case $CASE_NAME \ + --compset FPHYStest --res ne5_ne5_mg37 --run-unsupported WORKDIR $CASE_NAME @@ -95,15 +81,9 @@ RUN ./case.setup RUN ./xmlchange CAM_CONFIG_OPTS="--dyn none --physics-suites musica" RUN ./xmlchange CAM_LINKED_LIBS="-lmusica-fortran -lmusica -lyaml-cpp" +RUN ./xmlchange CAM_LINKED_LIBS="-L/home/cam_sima_user/scratch/test-case/bld/atm/obj/musica/lib -lmusica-fortran -lmusica -lyaml-cpp" RUN ./xmlchange ROF_NCPL=48 RUN ./xmlchange STOP_OPTION=nsteps RUN ./xmlchange STOP_N=5 -# Copy in the grid files and a snapshot file -RUN chmod +x /home/cam_sima_user/CAM-SIMA/docker/ftp_download.sh -RUN /home/cam_sima_user/CAM-SIMA/docker/ftp_download.sh - -# # add the snapshot file -RUN echo "ncdata='/home/cam_sima_user/run_heldsuarez_cam6_nt2_bigg_try005.cam.h5.0001-01-01-00000.nc'" >> user_nl_cam - RUN ./case.build diff --git a/src/physics/utils/musica_ccpp_dependencies.meta b/src/physics/utils/musica_ccpp_dependencies.meta index c514b661..c7ceb258 100644 --- a/src/physics/utils/musica_ccpp_dependencies.meta +++ b/src/physics/utils/musica_ccpp_dependencies.meta @@ -30,7 +30,7 @@ protected = True [ surface_albedo ] standard_name = surface_albedo_due_to_UV_and_VIS_direct - units = none + units = fraction type = real | kind = kind_phys dimensions = (horizontal_dimension) protected = True