Skip to content

Commit

Permalink
Add cades-baseline support, remove old machines
Browse files Browse the repository at this point in the history 
Fix deprecated arguments to xmlchange
call preview_namelist from correct location (updated in CIME)
  • Loading branch information
Daniel Ricciuto committed Feb 22, 2024
1 parent 63bfd4b commit ca01781
Show file tree
Hide file tree
Showing 3 changed files with 58 additions and 159 deletions.
9 changes: 6 additions & 3 deletions global_fullrun.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -801,7 +801,10 @@ def get_regional_bounds(myregion):
output.write('#SBATCH --qos=regular\n')
if ('compy' in options.machine and options.debug):
output.write('#SBATCH -p short\n')
if ('cades' in options.machine):
if ('cades-baseline' in options.machine):
output.write('#SBATCH -A CLI185\n')
output.write('#SBATCH -p batch\n')
elif ('cades' in options.machine):
output.write('#SBATCH -A ccsi\n')
output.write('#SBATCH -p batch\n')
output.write('#SBATCH --mem=64G\n')
Expand All @@ -813,8 +816,8 @@ def get_regional_bounds(myregion):

if (options.machine == 'cades'):
output.write('source $MODULESHOME/init/bash\n')
output.write('module unload python\n')
output.write('module load python/2.7.12\n\n')
#output.write('module unload python\n')
#output.write('module load python/2.7.12\n\n')
if (options.machine == 'eos'):
output.write('source $MODULESHOME/init/csh\n')
output.write('module load nco\n')
Expand Down
162 changes: 42 additions & 120 deletions runcase.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -376,23 +376,12 @@

#machine info: cores per node
ppn=1
if ('titan' in options.machine):
ppn=16
if (int(options.walltime) > 2 and int(options.ng) < 2048):
print('Requested walltime too long')
print('Setting to 2 hours.')
options.walltime=2
elif ('metis' in options.machine):
ppn=16
elif ('oic2' in options.machine):
ppn=8
elif ('oic5' in options.machine or 'cori-haswell' in options.machine or 'eos' in options.machine \
or 'cades' in options.machine):
if ('cades-baseline' in options.machine):
ppn=128
elif ('cori-haswell' in options.machine or 'cades' in options.machine):
ppn=32
elif ('cori-knl' in options.machine):
ppn=64
elif ('edison' in options.machine):
ppn=24
elif ('anvil' in options.machine):
ppn=36
elif ('compy' in options.machine):
Expand Down Expand Up @@ -631,9 +620,6 @@
#Construct case build and run directory
if (options.exeroot == '' or (os.path.exists(options.exeroot) == False)):
exeroot = runroot+'/'+casename+'/bld'
#if ('titan' in options.machine or 'eos' in options.machine):
# exeroot = os.path.abspath(os.environ['HOME']+ \
# '/acme_scratch/pointclm/'+casename+'/bld')
else:
options.no_build=True
exeroot=options.exeroot
Expand Down Expand Up @@ -699,47 +685,11 @@
if(options.humhol):
ptcmd = ptcmd + ' --humhol'

if (options.machine == 'eos' or options.machine == 'titan'):
os.system('rm temp/*.nc')
print('Note: NCO operations are slow on eos and titan.')
print('Submitting PBS script to make surface and domain data on rhea')
pbs_rhea=open('makepointdata_rhea.pbs','w')
pbs_rhea.write('#PBS -l walltime=00:30:00\n')
pbs_rhea.write('#PBS -l nodes=1\n')
pbs_rhea.write('#PBS -A cli112\n')
pbs_rhea.write('#PBS -q rhea\n')
pbs_rhea.write('#PBS -l gres=atlas1%atlas2\n\n')
pbs_rhea.write(' module load nco\n')
pbs_rhea.write(' cd '+os.getcwd()+'\n')
pbs_rhea.write(' rm temp/*.nc\n')
pbs_rhea.write(' module unload PE-intel\n')
pbs_rhea.write(' module load PE-gnu\n')
pbs_rhea.write(' module load python\n')
pbs_rhea.write(' module load python_numpy\n')
pbs_rhea.write(' module load python_scipy\n')
pbs_rhea.write(ptcmd+'\n')
pbs_rhea.close()
os.system('qsub makepointdata_rhea.pbs')
n_nc_files = 3
if (options.nopftdyn):
n_nc_files = 2
n=0
while (n < n_nc_files):
#Wait until files have been generated on rhea to proceed
list_dir = os.listdir('./temp')
n=0
for file in list_dir:
if file.endswith('.nc'):
n=n+1
os.system('sleep 10')
#Clean up
os.system('rm makepointdata_rhea*')
else:
print(ptcmd)
result = os.system(ptcmd)
if (result > 0):
print ('PointCLM: Error creating point data. Aborting')
sys.exit(1)
print(ptcmd)
result = os.system(ptcmd)
if (result > 0):
print ('PointCLM: Error creating point data. Aborting')
sys.exit(1)

if(options.makepointdata_only):
print ('PointCLM: Successfully creating point data ONLY, i.e. no further config/build/run CLM/ELM')
Expand Down Expand Up @@ -1131,11 +1081,11 @@
print('resetting maxpatch_pft to '+str(options.maxpatch_pft))
xval = subprocess.check_output('./xmlquery --value CLM_BLDNML_OPTS', cwd=casedir, shell=True)
xval = '-maxpft '+str(options.maxpatch_pft)+' '+xval
os.system("./xmlchange --id CLM_BLDNML_OPTS --val '" + xval + "'")
os.system("./xmlchange CLM_BLDNML_OPTS = '" + xval + "'")

# for spinup and transient runs, PIO_TYPENAME is pnetcdf, which now not works well
if('mac' in options.machine or 'cades' in options.machine):
os.system("./xmlchange --id PIO_TYPENAME --val netcdf ")
os.system("./xmlchange PIO_TYPENAME=netcdf ")


#--------------------------CESM setup ----------------------------------------
Expand Down Expand Up @@ -1594,7 +1544,7 @@

#configure case
#if (isglobal):
os.system("./xmlchange --id BATCH_SYSTEM --val none")
os.system("./xmlchange BATCH_SYSTEM=none")
if (options.no_config == False):
print('Running case.setup')
result = os.system('./case.setup > case_setup.log')
Expand All @@ -1608,14 +1558,14 @@
#Land CPPDEF modifications
if (options.humhol):
print("Turning on HUM_HOL modification\n")
os.system("./xmlchange --id "+mylsm+"_CONFIG_OPTS --append --val '-cppdefs -DHUM_HOL'")
os.system("./xmlchange --append "+mylsm+"_CONFIG_OPTS='-cppdefs -DHUM_HOL'")

if (options.marsh):
print("Turning on MARSH modification\n")
os.system("./xmlchange --id "+mylsm+"_CONFIG_OPTS --append --val '-cppdefs -DMARSH'")
os.system("./xmlchange --append "+mylsm+"_CONFIG_OPTS='-cppdefs -DMARSH'")
if (options.harvmod):
print('Turning on HARVMOD modification\n')
os.system("./xmlchange --id "+mylsm+"_CONFIG_OPTS --append --val '-cppdefs -DHARVMOD'")
os.system("./xmlchange --append "+mylsm+"_CONFIG_OPTS='-cppdefs -DHARVMOD'")

#Global CPPDEF modifications
if (cpl_bypass):
Expand Down Expand Up @@ -1676,11 +1626,8 @@
#compile cesm
if (options.no_build == False):
print('Running case.build')
if ('edison' in options.machine or 'titan' in options.machine):
#send output to screen since build times are very slow
result = os.system('./case.build')
else:
result = os.system('./case.build > case_build.log')
result = os.system('./case.build')
#result = os.system('./case.build > case_build.log')
if (result > 0):
print('Error: Pointclm.py failed to build case. Aborting')
print('See '+os.getcwd()+'/case_build.log for details')
Expand Down Expand Up @@ -1782,7 +1729,10 @@
myinput.close()
myoutput.close()
# run preview_namelists to copy user_datm.streams.... to CaseDocs
os.system(os.path.abspath(options.csmdir)+'/cime/scripts/Tools/preview_namelists')
if os.path.exists(os.path.abspath(options.csmdir)+'/cime/scripts/Tools/preview_namelists'):
os.system(os.path.abspath(options.csmdir)+'/cime/scripts/Tools/preview_namelists')
else:
os.system(os.path.abspath(options.csmdir)+'/cime/CIME/Tools/preview_namelists')


#copy site data to run directory
Expand Down Expand Up @@ -1849,7 +1799,7 @@
print('Error: ensemble file does not exist')
sys.exit(1)

samples=numpy.zeros((n_parameters,100000), dtype=numpy.float)
samples=numpy.zeros((n_parameters,100000), dtype=float)
#get parameter samples and information
myinput=open(options.ensemble_file)
nsamples = 0
Expand All @@ -1860,7 +1810,7 @@
myinput.close()
elif (int(options.mc_ensemble) > 0):
nsamples = int(options.mc_ensemble)
samples=numpy.zeros((n_parameters,nsamples), dtype=numpy.float)
samples=numpy.zeros((n_parameters,nsamples), dtype=float)
for i in range(0,nsamples):
for j in range(0,n_parameters):
samples[j][i] = param_min[j]+(param_max[j]-param_min[j])*numpy.random.rand(1)
Expand All @@ -1882,7 +1832,7 @@
#Launch ensemble if requested
mysubmit_type = 'qsub'
if ('cades' in options.machine or 'compy' in options.machine or 'ubuntu' in options.machine or 'cori' in options.machine or \
options.machine == 'anvil' or options.machine == 'edison' or options.machine == 'chrysalis'):
options.machine == 'anvil' or options.machine == 'chrysalis'):
mysubmit_type = 'sbatch'
if (options.ensemble_file != ''):
os.system('mkdir -p '+PTCLMdir+'/scripts/'+myscriptsdir)
Expand All @@ -1899,18 +1849,18 @@
output_run.write('#PBS -N ens_'+casename+'\n')
if (options.project != ''):
output_run.write('#PBS -A '+options.project+'\n')
if (options.machine == 'cades'):
output_run.write('#PBS -l nodes='+str(int(math.ceil(np_total/(ppn*1.0))))+ \
':ppn='+str(ppn)+'\n')
output_run.write('#PBS -W group_list=cades-ccsi\n')
else:
output_run.write('#PBS -l nodes='+str(int(math.ceil(np_total/(ppn*1.0))))+ \
#if (options.machine == 'cades'):
# output_run.write('#PBS -l nodes='+str(int(math.ceil(np_total/(ppn*1.0))))+ \
# ':ppn='+str(ppn)+'\n')
# output_run.write('#PBS -W group_list=cades-ccsi\n')
#else:
output_run.write('#PBS -l nodes='+str(int(math.ceil(np_total/(ppn*1.0))))+ \
'\n')
else:
output_run.write('#SBATCH --time='+timestr+'\n')
output_run.write('#SBATCH -J ens_'+casename+'\n')
output_run.write('#SBATCH --nodes='+str(int(math.ceil(np_total/(ppn*1.0))))+'\n')
if ('edison' in options.machine or 'cori' in options.machine):
if ('cori' in options.machine):
if (options.debug):
output_run.write('#SBATCH --qos=debug\n')
else:
Expand All @@ -1921,44 +1871,20 @@
output_run.write('#SBATCH --constraint=knl\n')
if ('compy' in options.machine and options.debug):
output_run.write('#SBATCH --qos=short\n')
if ('cades' in options.machine):
output_run.write('#SBATCH -A ccsi\n')
if ('cades-baseline' in options.machine):
output_run.write('#SBATCH -A CLI185\n')
output_run.write('#SBATCH -p batch\n')
output_run.write('#SBATCH --mem=64G\n')
output_run.write('#SBATCH --ntasks-per-node 32\n')
output_run.write('#SBATCH --ntasks-per-node 128\n')
elif ('cades' in options.machine):
output_run.write('#SBATCH -A ccsi\n')
output_run.write('#SBATCH -p batch\n')
output_run.write('#SBATCH --mem=64G\n')
output_run.write('#SBATCH --ntasks-per-node 32\n')
if ('anvil' in options.machine):
output_run.write('#SBATCH -A condo\n')
output_run.write('#SBATCH -p acme-small\n')
output_run.write("\n")
if (options.machine == 'eos'):
output_run.write('source $MODULESHOME/init/csh\n')
output_run.write('module load nco\n')
output_run.write('module load cray-netcdf\n')
output_run.write('module unload python\n')
output_run.write('module load python/2.7.5\n')
output_run.write('module unload PrgEnv-intel\n')
output_run.write('module load PrgEnv-gnu\n')
output_run.write('module load python_numpy\n')
output_run.write('module load python_scipy\n')
output_run.write('module load python_mpi4py/2.0.0\n')
output_run.write('module unload PrgEnv-gnu\n')
output_run.write('module load PrgEnv-intel\n')
if (options.machine == 'titan'):
output_run.write('source $MODULESHOME/init/csh\n')
output_run.write('module load nco\n')
output_run.write('module load cray-netcdf\n')
output_run.write('module load python/2.7.9\n')
output_run.write('module load python_numpy/1.9.2\n')
output_run.write('module load python_scipy/0.15.1\n')
output_run.write('module load python_mpi4py/2.0.0\n')
if ('cori' in options.machine or 'edison' in options.machine):
output_run.write('module unload python\n')
output_run.write('module unload scipy\n')
output_run.write('module unload numpy\n')
output_run.write('module load cray-netcdf\n')
output_run.write('module load python/2.7-anaconda-5.2\n')
output_run.write('module load nco\n')
if ('compy' in options.machine or 'anvil' in options.machine or 'chrysalis' in options.machine):
if ('cades' in options.machine or 'compy' in options.machine or 'anvil' in options.machine or 'chrysalis' in options.machine):
#get the software environment
softenvfile = open(casedir+'/software_environment.txt','r')
for line in softenvfile:
Expand All @@ -1972,13 +1898,9 @@
if ('docker' in options.machine or 'oic' in options.machine or 'cades' in options.machine or 'ubuntu' in options.machine):
mpicmd = 'mpirun'
if ('cades' in options.machine):
mpicmd = '/software/dev_tools/swtree/cs400_centos7.2_pe2016-08/openmpi/1.10.3/centos7.2_gnu5.3.0/bin/mpirun'
cmd = mpicmd+' -np '+str(np_total)+' python manage_ensemble.py ' \
+'--case '+casename+' --runroot '+runroot+' --n_ensemble '+str(nsamples)+' --ens_file '+ \
options.ensemble_file+' --exeroot '+exeroot+' --parm_list '+options.parm_list+' --cnp '+cnp + \
' --site '+options.site+' --model_name '+model_name
elif (('titan' in options.machine or 'eos' in options.machine) and int(options.ninst) == 1):
cmd = 'aprun -n '+str(np_total)+' python manage_ensemble.py ' \
#mpicmd = '/software/dev_tools/swtree/cs400_centos7.2_pe2016-08/openmpi/1.10.3/centos7.2_gnu5.3.0/bin/mpirun'
mpicmd = 'srun'
cmd = mpicmd+' -n '+str(np_total)+' python manage_ensemble.py ' \
+'--case '+casename+' --runroot '+runroot+' --n_ensemble '+str(nsamples)+' --ens_file '+ \
options.ensemble_file+' --exeroot '+exeroot+' --parm_list '+options.parm_list+' --cnp '+cnp + \
' --site '+options.site+' --model_name '+model_name
Expand Down
46 changes: 10 additions & 36 deletions site_fullrun.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -327,8 +327,6 @@ def submit(fname, submit_type='qsub', job_depend=''):

if (options.ccsm_input != ''):
ccsm_input = options.ccsm_input
elif (options.machine == 'titan' or options.machine == 'eos'):
ccsm_input = '/lustre/atlas/world-shared/cli900/cesm/inputdata'
elif (options.machine == 'cades'):
ccsm_input = '/nfs/data/ccsi/proj-shared/E3SM/inputdata/'
elif (options.machine == 'edison' or 'cori' in options.machine):
Expand All @@ -339,9 +337,7 @@ def submit(fname, submit_type='qsub', job_depend=''):
ccsm_input = '/compyfs/inputdata/'

#if (options.compiler != ''):
# if (options.machine == 'titan'):
# options.compiler = 'pgi'
# if (options.machine == 'eos' or options.machine == 'edison' or 'cori' in options.machine):
# if ('cori' in options.machine):
# options.compiler = 'intel'
# if (options.machine == 'cades'):
# options.compiler = 'gnu'
Expand All @@ -368,21 +364,14 @@ def submit(fname, submit_type='qsub', job_depend=''):
myproject='e3sm'
if (options.runroot == '' or (os.path.exists(options.runroot) == False)):
myuser = getpass.getuser()
if (options.machine == 'titan' or options.machine == 'eos'):
myinput = open('/ccs/home/'+myuser+'/.cesm_proj','r')
for s in myinput:
myproject=s[:-1]
runroot='/lustre/atlas/scratch/'+myuser+'/'+myproject
elif (options.machine == 'cades'):
if (options.machine == 'cades'):
runroot='/lustre/or-scratch/cades-ccsi/scratch/'+myuser
elif ('cori' in options.machine):
runroot='/global/cscratch1/sd/'+myuser
myinput = open(os.environ.get('HOME')+'/.cesm_proj','r')
for s in myinput:
myproject=s[:-1]
print('Project = '+myproject)
elif ('edison' in options.machine):
runroot=os.environ.get('CSCRATCH')+'/acme_scratch/edison/'
elif ('anvil' in options.machine or 'chrysalis' in options.machine):
runroot="/lcrc/group/acme/"+myuser
myproject='e3sm'
Expand Down Expand Up @@ -1123,11 +1112,15 @@ def submit(fname, submit_type='qsub', job_depend=''):
output.write('#SBATCH --partition=debug\n')
else:
output.write('#SBATCH --partition=regular\n')
if ('cades' in options.machine):
if ('cades-baseline' in options.machine):
output.write('#SBATCH -A CLI185\n')
output.write('#SBATCH -p batch\n')
output.write('#SBATCH --ntasks-per-node 128\n')
elif ('cades' in options.machine):
output.write('#SBATCH -A ccsi\n')
output.write('#SBATCH -p batch\n')
output.write('#SBATCH --mem='+str(npernode*2)+'G\n')
output.write('#SBATCH --ntasks-per-node '+str(npernode)+'\n')
output.write('#SBATCH --mem=64G\n')
output.write('#SBATCH --ntasks-per-node 32\n')
elif ("#" in s and "ppn" in s):
if ('cades' in options.machine):
#if ('diags' in c or 'iniadjust' in c):
Expand All @@ -1145,26 +1138,7 @@ def submit(fname, submit_type='qsub', job_depend=''):
input.close()
output.write("\n")

if (options.machine == 'eos'):
output.write('source $MODULESHOME/init/csh\n')
output.write('module load nco\n')
output.write('module unload python\n')
output.write('module load python/2.7.5\n')
output.write('module unload PrgEnv-intel\n')
output.write('module load PrgEnv-gnu\n')
output.write('module load python_numpy\n')
output.write('module load python_scipy\n')
output.write('module load python_mpi4py/2.0.0\n')
output.write('module unload PrgEnv-gnu\n')
output.write('module load PrgEnv-intel\n')
if (options.machine == 'titan'):
output.write('source $MODULESHOME/init/csh\n')
output.write('module load nco\n')
output.write('module load python\n')
output.write('module load python_numpy/1.9.2\n')
output.write('module load python_scipy/0.15.1\n')
output.write('module load python_mpi4py/2.0.0\n')
if (options.machine == 'edison' or 'cori' in options.machine):
if ('cori' in options.machine):
output.write('source $MODULESHOME/init/csh\n')
output.write('module unload python\n')
output.write('module unload scipy\n')
Expand Down

0 comments on commit ca01781

Please sign in to comment.