Skip to content

Commit

Permalink
Support for enabling optional MPAS features through CIME (#351)
Browse files Browse the repository at this point in the history 
### Tag name (required for release branches):

None

### Originator(s):

kuanchihwang

### Descriptions (include the issue title, and the keyword ['closes',
'fixes', 'resolves'] followed by the issue number):

This PR refactors the build system of MPAS dycore. Build performance has
been improved. PIO version selection is now more robust. The
availability of optional MPAS features is now determined by querying
relevant configuration options from CIME rather than being hard-coded.

MPAS dycore is in double precision mode by default. Users can choose to
enable single precision mode by running:

```bash
./xmlchange --append CAM_CONFIG_OPTS="--dyn-kind REAL32"
```

Closes #347.

### Describe any changes made to the build system:

See above.

### Describe any changes made to the namelist:

None

### List any changes to the defaults for the input datasets (e.g.,
boundary datasets):

None

### List all files eliminated and why:

None

### List all files added and what they do:

None

### List all existing files that have been modified, and describe the
changes:

* `M       cime_config/buildlib`
  * Support for enabling optional MPAS features through CIME
  * Remove trailing spaces
* `M       src/dynamics/mpas/assets/Makefile.in.CESM`
  * Avoid using recursively expanded variables
  * Make PIO version selection more robust
  * Support for enabling optional MPAS features through CIME

### Regression tests:

No changes to any existing tests with respect to the last baseline,
`sima0_03_000`.

*
SMS_Ln9.ne5pg3_ne5pg3_mg37.FCAM7.derecho_gnu.cam-outfrq_se_cslam_analy_ic
(Overall: FAIL)
*
SMS_Ln9.ne5pg3_ne5pg3_mg37.FCAM7.derecho_intel.cam-outfrq_se_cslam_analy_ic
(Overall: FAIL)
  * Existing test failures
  • Loading branch information
@kuanchihwang
kuanchihwang authored Feb 20, 2025
1 parent 8b4d541 commit 1cd2736
Show file tree
Hide file tree
Showing 2 changed files with 87 additions and 52 deletions.
30 changes: 21 additions & 9 deletions cime_config/buildlib
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,28 +147,40 @@ def _build_cam():
# build the library
#-------------------------------------------------------

# If dynamical core is MPAS, setup its build infrastructure so it can be built along with CAM below.
if dycore == "mpas":
_setup_mpas(case)

complib = os.path.join(libroot, "libatm.a")
makefile = os.path.join(casetools, "Makefile")

cmd = f"{gmake} complib -j {gmake_j} MODEL=cam COMPLIB={complib}"
cmd += f" -f {makefile} {get_standard_makefile_args(case)} "
cmd += f" -f {makefile} {get_standard_makefile_args(case)}"

# Add C Pre-Processor (CPP) definitions, if present:
if config.cpp_defs:
ccpp_defs_str = ' '.join(config.cpp_defs)
cmd += f" USER_CPPDEFS='{ccpp_defs_str}'"

# If dynamical core is MPAS, prepare its build infrastructure so it can be built along with CAM-SIMA below.
if dycore == "mpas":
_prepare_mpas(case)

optional_mpas_features = []

if case.get_value("USE_ESMF_LIB"):
optional_mpas_features.append("+ESMF_LIBRARY")
optional_mpas_features.append("+PERF_MOD_LIBRARY")

if config.get_value("dyn_kind") == "REAL32":
optional_mpas_features.append("+SINGLE_PRECISION")

if len(optional_mpas_features) > 0:
cmd += " OPTIONAL_MPAS_FEATURES=\"" + " ".join(optional_mpas_features) + "\""

retcode, out, err = run_cmd(cmd)
_LOGGER.info("%s: \n\n output:\n %s \n\n err:\n\n%s\n", cmd, out, err)
_LOGGER.info("Command %s:\n\nstdout:\n%s\n\nstderr:\n%s\n", cmd, out, err)
expect(retcode == 0, f"Command {cmd} failed with rc={retcode}")

def _setup_mpas(case: Case) -> None:
def _prepare_mpas(case: Case) -> None:
"""
Setup MPAS build infrastructure.
Prepare MPAS build infrastructure.
"""

atm_src_root = os.path.normpath(case.get_value("COMP_ROOT_DIR_ATM"))
Expand All @@ -177,7 +189,7 @@ def _setup_mpas(case: Case) -> None:
mpas_dycore_src_root = os.path.join(atm_src_root, "src", "dynamics", "mpas", "dycore", "src")
mpas_dycore_bld_root = os.path.join(atm_bld_root, "mpas")

# Make sure `mpas_dycore_src_root` exists. If not, it is likely that `./manage_externals/checkout_externals` did not succeed.
# Make sure `mpas_dycore_src_root` exists. If not, it is likely that `./bin/git-fleximod update` did not succeed.
if os.path.isfile(mpas_dycore_src_root):
raise FileExistsError(1, "Unexpected file", mpas_dycore_src_root)

Expand Down
109 changes: 66 additions & 43 deletions src/dynamics/mpas/assets/Makefile.in.CESM
View file
Original file line number Diff line number Diff line change
@@ -1,62 +1,82 @@
# `LIBROOT` is defined by CIME under normal circumstances.
ifeq ($(strip $(LIBROOT)),)
$(warning `LIBROOT` should not be empty. Defaulting to `..`)

LIBROOT = ..
LIBROOT := ..
endif

# `SRCROOT` is defined by CIME under normal circumstances.
ifeq ($(strip $(SRCROOT)),)
$(warning `SRCROOT` should not be empty. Defaulting to `..`)

SRCROOT = ..
MPAS_SRC_ROOT = $(SRCROOT)
SRCROOT := ..
MPAS_SRC_ROOT := $(SRCROOT)
else
MPAS_SRC_ROOT = $(SRCROOT)/src/dynamics/mpas/dycore
MPAS_SRC_ROOT := $(SRCROOT)/src/dynamics/mpas/dycore
endif

# `PIO_VERSION` is defined by CIME under normal circumstances.
ifeq ($(filter 1 2,$(PIO_VERSION)),)
$(warning `PIO_VERSION` should have a value of `1` or `2`. Defaulting to `2`)

PIO_VERSION := 2
endif

#
# Define and export variables used by MPAS build infrastructure.
#

export CP = cp -afv
export LN = ln -fsv
export MKDIR = mkdir -pv
export RM = rm -frv
export CP := cp -afv
export LN := ln -fsv
export MKDIR := mkdir -pv
export RM := rm -frv

# Constants.
export AUTOCLEAN = false
export BUILD_TARGET = N/A
export CORE = atmosphere
export EXE_NAME = atmosphere_model
export GEN_F90 = false
export GIT_VERSION = $(shell git -C "$(MPAS_SRC_ROOT)" describe --always --dirty --tags || echo "N/A")
export NAMELIST_SUFFIX = atmosphere
export AUTOCLEAN := false
export BUILD_TARGET := N/A
export CORE := atmosphere
export EXE_NAME := atmosphere_model
export GEN_F90 := false
export GIT_VERSION := $(shell git -C "$(MPAS_SRC_ROOT)" describe --always --dirty --tags || echo "N/A")
export NAMELIST_SUFFIX := atmosphere

# Customize variables (e.g., build options) for use with CESM.
export AR := ar
export ARFLAGS := -M
export CPP := cpp -P -traditional
export CPPFLAGS := -D_MPI \
-DMPAS_BUILD_TARGET="$(BUILD_TARGET)" \
-DMPAS_CAM_DYCORE \
-DMPAS_EXE_NAME="$(EXE_NAME)" \
-DMPAS_EXTERNAL_ESMF_LIB \
-DMPAS_GIT_VERSION="$(GIT_VERSION)" \
-DMPAS_NAMELIST_SUFFIX="$(NAMELIST_SUFFIX)" \
-DMPAS_NATIVE_TIMERS \
-DMPAS_NO_ESMF_INIT \
-DMPAS_PIO_SUPPORT
# `PIODEF` is defined by CIME Makefile (i.e., `cime/CIME/Tools/Makefile`). Its value can be empty or `-DUSE_PIO2`.
ifneq ($(strip $(PIODEF)),)
export CPPFLAGS += $(strip $(PIODEF))
export AR := ar
export ARFLAGS := -M
export CPP := cpp -P -traditional
export CPPFLAGS := -D_MPI \
-DMPAS_BUILD_TARGET="$(BUILD_TARGET)" \
-DMPAS_CAM_DYCORE \
-DMPAS_EXE_NAME="$(EXE_NAME)" \
-DMPAS_GIT_VERSION="$(GIT_VERSION)" \
-DMPAS_NAMELIST_SUFFIX="$(NAMELIST_SUFFIX)" \
-DMPAS_PIO_SUPPORT
ifeq ($(strip $(PIO_VERSION)),2)
export CPPFLAGS += -DUSE_PIO2
endif
export LINKER := $(strip $(FC))
export SCC := $(strip $(CC))
export SCXX := $(strip $(CXX))
export SFC := $(strip $(FC))

# Check for optional features.
is-enabled = $(if $(filter +$(1),$(sort $(strip $(OPTIONAL_MPAS_FEATURES)))),true,false)

ifeq ($(call is-enabled,ESMF_LIBRARY),true)
export CPPFLAGS += -DMPAS_EXTERNAL_ESMF_LIB \
-DMPAS_NO_ESMF_INIT
endif
ifeq ($(call is-enabled,MPI_F08_INTERFACE),true)
export CPPFLAGS += -DMPAS_USE_MPI_F08
endif
ifeq ($(call is-enabled,PERF_MOD_LIBRARY),true)
export CPPFLAGS += -DMPAS_PERF_MOD_TIMERS
else
export CPPFLAGS += -DMPAS_NATIVE_TIMERS
endif
ifeq ($(call is-enabled,SINGLE_PRECISION),true)
export CPPFLAGS += -DSINGLE_PRECISION
endif
# Uncomment below for MPI Fortran 2008 interface support. There is currently no corresponding option in CIME to enable this.
# export CPPFLAGS += -DMPAS_USE_MPI_F08
# Uncomment below for single precision mode support. There is currently no corresponding option in CIME to enable this.
# export CPPFLAGS += -DSINGLE_PRECISION
export LINKER := $(strip $(FC))
export SCC := $(strip $(CC))
export SCXX := $(strip $(CXX))
export SFC := $(strip $(FC))

#
# Targets.
Expand All @@ -67,12 +87,12 @@ all:
@echo 'Supplemental Makefile for MPAS Dynamical Core in CESM'
@echo ''
@echo 'MPAS will be built as a static library located at `$${LIBROOT}/libmpas.a`.'
@echo 'Users are responsible to provide all necessary build options via environment variables or command line arguments.'
@echo 'Users are responsible for providing all necessary build options via environment variables or command line arguments.'
@echo ''
@echo 'Usage hints:'
@echo ' `make libmpas-prepare ESM="CESM" LIBROOT="..." SRCROOT="..."`'
@echo ' `make libmpas-build ESM="CESM" LIBROOT="..." SRCROOT="..."`'
@echo ' `make libmpas-clean ESM="CESM" LIBROOT="..." SRCROOT="..."`'
@echo ' `make libmpas-prepare ESM="CESM" LIBROOT="..." SRCROOT="..." PIO_VERSION="1|2"`'
@echo ' `make libmpas-build ESM="CESM" LIBROOT="..." SRCROOT="..." PIO_VERSION="1|2"`'
@echo ' `make libmpas-clean ESM="CESM" LIBROOT="..." SRCROOT="..." PIO_VERSION="1|2"`'

.PHONY: libmpas-prepare
libmpas-prepare: libmpas-apply-patch libmpas-archiver-script.txt libmpas-preview
Expand Down Expand Up @@ -135,6 +155,9 @@ libmpas-clean: clean

.PHONY: externals
externals: $(AUTOCLEAN_DEPS)
ifeq ($(call is-enabled,ESMF_LIBRARY),false)
( cd external; $(MAKE) FC="$(FC)" SFC="$(SFC)" CC="$(CC)" SCC="$(SCC)" FFLAGS="$(FFLAGS)" CFLAGS="$(CFLAGS)" CPP="$(CPP)" NETCDF="$(NETCDF)" CORE="$(CORE)" esmf_time )
endif
( cd external; $(MAKE) FC="$(FC)" SFC="$(SFC)" CC="$(CC)" SCC="$(SCC)" FFLAGS="$(FFLAGS)" CFLAGS="$(CFLAGS)" CPP="$(CPP)" NETCDF="$(NETCDF)" CORE="$(CORE)" ezxml-lib )

.PHONY: subdrv
Expand Down

0 comments on commit 1cd2736

Please sign in to comment.