From 14d952d985d29260d06285f830f052ca49c424ca Mon Sep 17 00:00:00 2001 From: kdraeder Date: Fri, 27 Mar 2026 15:50:05 -0600 Subject: [PATCH] Made coarse-resolution large-ensemble MCC, ERI tests work. Changes are needed in multiple components: cesm, cime, cmeps, mom/MOM6. The branches are labeled with DART_lowres_{component}. cime_config/config_compsets.xml Added BHIST_LT_DART compset. cime_config/testlist_allactive.xml Added coarse resolution ERI and MCC tests for BHIST DART tests cime_config/testmods_dirs/allactive/DART_BHIST_lowres/ Testmod to make coarse resolution BHIST tests work. Must be used with the (new) MOM6 testmod: mom-tx10deg + include_user_mods (<- ./crossleap <- ./defaultio) + shell_commands; fix or sidestep several barriers to successful testing + user_nl_cam; namelist settings required by DART. + user_nl_clm; permit mid-year start dates in nondefault years. + user_nl_mosart; provide a frivinp file which works with the coarse resolution. + README_layout; instructions for setting NINST and MAX_TASKS_PER_NODE to enable running. Possibly not needed in the future. --- cime_config/config_compsets.xml | 6 ++ cime_config/testlist_allactive.xml | 19 ++++++ .../allactive/DART_BHIST_lowres/README_layout | 62 +++++++++++++++++++ .../DART_BHIST_lowres/include_user_mods | 1 + .../DART_BHIST_lowres/shell_commands | 7 +++ .../allactive/DART_BHIST_lowres/user_nl_cam | 1 + .../allactive/DART_BHIST_lowres/user_nl_clm | 2 + .../DART_BHIST_lowres/user_nl_mosart | 1 + 8 files changed, 99 insertions(+) create mode 100644 cime_config/testmods_dirs/allactive/DART_BHIST_lowres/README_layout create mode 100644 cime_config/testmods_dirs/allactive/DART_BHIST_lowres/include_user_mods create mode 100644 cime_config/testmods_dirs/allactive/DART_BHIST_lowres/shell_commands create mode 100644 cime_config/testmods_dirs/allactive/DART_BHIST_lowres/user_nl_cam create mode 100644 cime_config/testmods_dirs/allactive/DART_BHIST_lowres/user_nl_clm create mode 100644 cime_config/testmods_dirs/allactive/DART_BHIST_lowres/user_nl_mosart diff --git a/cime_config/config_compsets.xml b/cime_config/config_compsets.xml index 2c060884ad..1b81f68512 100644 --- a/cime_config/config_compsets.xml +++ b/cime_config/config_compsets.xml @@ -92,6 +92,12 @@ HISTC_CAM70%LT_CLM60%BGC-CROP_CICE_MOM6_MOSART_DGLC%NOEVOLVE_WW3 + + + BHISTC_LT_DART + HISTC_CAM70%LT_CLM60%BGC_CICE_MOM6_MOSART_DGLC%NOEVOLVE_SWAV + diff --git a/cime_config/testlist_allactive.xml b/cime_config/testlist_allactive.xml index 95a993f4f8..9ebd7ee7bb 100644 --- a/cime_config/testlist_allactive.xml +++ b/cime_config/testlist_allactive.xml @@ -73,6 +73,25 @@ + + + + + + + + + + + + + + + + + diff --git a/cime_config/testmods_dirs/allactive/DART_BHIST_lowres/README_layout b/cime_config/testmods_dirs/allactive/DART_BHIST_lowres/README_layout new file mode 100644 index 0000000000..7103c12be1 --- /dev/null +++ b/cime_config/testmods_dirs/allactive/DART_BHIST_lowres/README_layout @@ -0,0 +1,62 @@ +If the coarse resolution (ne3pg3_ne3pg3_t232) is used in a BHIST MCC test, +then the number of instances and MAX_TASKS_PER_NODE must be chosen +in a way that's consistent with how cime set the PE layout. + +1) MOM6 coarse resolution is limited (2026-3) to 6 tasks. + The SE dycore is limited to 6 x ne^2 tasks, + so the ne3 grid could be given up to 54 tasks. + drv-tx10deg/shell_commands:ROOTPE_OCN assumes that only 6 will be given + to the other components, and then 6 to ocn, so tasks_per_inst = 12. + If more tasks are given to the other components, + then ROOTPE_OCN must be changed to that number. +2) If tasks_per_inst = 12, NINST <=10, and there are 128 PEs/node, + then the job will fit in 1 node and be run in derecho's develop queue. + The default PE layout will work. + You can stop here. + For NINST > 10 the job requires multiple nodes and the default layout + does not work because at least one instance will be split onto 2 nodes. + During testing these jobs never finished, despite trying their best. +3) The minimum number of nodes required + min_nodes = int[(NINST * tasks_per_inst) / PEs_per_node] +1. + For NINST = 14 on derecho (128 PEs/node), + min_nodes = 2 +4) Cime calculates PEs/instance from + PEs_per_inst = (PEs_per_node * min_nodes) / NINST + = 17 +5) Whole instances must be assigned to each node. + That is, no instance can be split between 2 nodes. + In this example, put + NINST_per_node = int(NINST / min_nodes) + = 7 instances + on each node by setting + MAX_TASKS_PER_NODE = NINST_per_node * PEs_per_inst + = 7 * 17 = 119. + in ./shell_commands. +6) If your chosen NINST doesn't divide evenly by min_nodes, + then there will be "left over" instances after NINST_per_node * min_nodes + instances have been distributed. + They might fit into the last node. + If they don't, you'll need to add a node to the job request. + Alternatively, change NINST by a small number to make it divisible and try again, + starting at 3). +7) Use NINST in the test modifier _C{NINST} of the test name + +Assuming each instance gets 6 tasks for ocn and 6 for the other components: +NINST = 15; + min_nodes = [15 * 12 / 128] + 1 = 2 + PEs_per_inst = 128 * 2 / 15 = 17 + NINST_per_node = int(15 / 2) = 7 with 1 instance left over +> MAX_TASKS_PER_NODE = 7 * 17 = 119 + 128 - 119 = 9 PEs are available in the last node, + which is not enough for the leftover instance, so an additional node is needed. +NINST = 40; + min_nodes = 40 * 12 / 128 = 4 + PEs_per_inst = 128 * 4 / 40 = 12 + NINST_per_node = 40 / 4 = 10 +> MAX_TASKS_PER_NODE = 10 * 12 = 120 +NINST = 80; + min_nodes = 80 * 12 / 128 = 8 + PEs_per_inst = 128 * 8 / 80 = 12 + NINST_per_node = 80 / 8 = 10 +> MAX_TASKS_PER_NODE = 10 * 12 = 120 + diff --git a/cime_config/testmods_dirs/allactive/DART_BHIST_lowres/include_user_mods b/cime_config/testmods_dirs/allactive/DART_BHIST_lowres/include_user_mods new file mode 100644 index 0000000000..ae8286cb1b --- /dev/null +++ b/cime_config/testmods_dirs/allactive/DART_BHIST_lowres/include_user_mods @@ -0,0 +1 @@ +../crossleap diff --git a/cime_config/testmods_dirs/allactive/DART_BHIST_lowres/shell_commands b/cime_config/testmods_dirs/allactive/DART_BHIST_lowres/shell_commands new file mode 100644 index 0000000000..4a0b8a94ba --- /dev/null +++ b/cime_config/testmods_dirs/allactive/DART_BHIST_lowres/shell_commands @@ -0,0 +1,7 @@ +driver=`./xmlquery --value COMP_INTERFACE` +if [ "$driver" = "nuopc" ]; then + ./xmlchange GLC_NCPL=4 +fi + +./xmlchange JOB_PRIORITY=premium +./xmlchange MAX_TASKS_PER_NODE=120 diff --git a/cime_config/testmods_dirs/allactive/DART_BHIST_lowres/user_nl_cam b/cime_config/testmods_dirs/allactive/DART_BHIST_lowres/user_nl_cam new file mode 100644 index 0000000000..efd197aa72 --- /dev/null +++ b/cime_config/testmods_dirs/allactive/DART_BHIST_lowres/user_nl_cam @@ -0,0 +1 @@ +inithist='ENDOFRUN' diff --git a/cime_config/testmods_dirs/allactive/DART_BHIST_lowres/user_nl_clm b/cime_config/testmods_dirs/allactive/DART_BHIST_lowres/user_nl_clm new file mode 100644 index 0000000000..1623469d79 --- /dev/null +++ b/cime_config/testmods_dirs/allactive/DART_BHIST_lowres/user_nl_clm @@ -0,0 +1,2 @@ +check_finidat_year_consistency = .false. +for_testing_allow_non_annual_changes = .true. diff --git a/cime_config/testmods_dirs/allactive/DART_BHIST_lowres/user_nl_mosart b/cime_config/testmods_dirs/allactive/DART_BHIST_lowres/user_nl_mosart new file mode 100644 index 0000000000..9a24bab86a --- /dev/null +++ b/cime_config/testmods_dirs/allactive/DART_BHIST_lowres/user_nl_mosart @@ -0,0 +1 @@ +frivinp = "/glade/campaign/cesm/cesmdata/inputdata/rof/mosart/MOSART_routing_Global_0.5x0.5_c170601.nc"