From 5ae3e93aefe1664e51e1224e4892f9ee517f0a17 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Mon, 19 Jan 2026 13:33:49 +0100 Subject: [PATCH 01/47] Make CMOR tables configurable through new configuration system --- doc/reference/cmor_tables.rst | 4 +- esmvalcore/cmor/table.py | 594 ++++-- .../CMOR_MP_BC_tot.dat | 0 .../CMOR_MP_CFCl3.dat | 0 .../{custom => cmip5-custom}/CMOR_MP_CH4.dat | 0 .../{custom => cmip5-custom}/CMOR_MP_CO.dat | 0 .../{custom => cmip5-custom}/CMOR_MP_CO2.dat | 0 .../{custom => cmip5-custom}/CMOR_MP_ClOX.dat | 0 .../CMOR_MP_DU_tot.dat | 0 .../{custom => cmip5-custom}/CMOR_MP_N2O.dat | 0 .../{custom => cmip5-custom}/CMOR_MP_NH3.dat | 0 .../{custom => cmip5-custom}/CMOR_MP_NO.dat | 0 .../{custom => cmip5-custom}/CMOR_MP_NO2.dat | 0 .../{custom => cmip5-custom}/CMOR_MP_NOX.dat | 0 .../{custom => cmip5-custom}/CMOR_MP_O3.dat | 0 .../{custom => cmip5-custom}/CMOR_MP_OH.dat | 0 .../{custom => cmip5-custom}/CMOR_MP_S.dat | 0 .../{custom => cmip5-custom}/CMOR_MP_SO2.dat | 0 .../CMOR_MP_SO4mm_tot.dat | 0 .../CMOR_MP_SS_tot.dat | 0 .../{custom => cmip5-custom}/CMOR_agb.dat | 0 .../{custom => cmip5-custom}/CMOR_alb.dat | 0 .../{custom => cmip5-custom}/CMOR_albDiff.dat | 0 .../CMOR_albDiffiTr13.dat | 0 .../{custom => cmip5-custom}/CMOR_amoc.dat | 0 .../{custom => cmip5-custom}/CMOR_asr.dat | 0 .../{custom => cmip5-custom}/CMOR_awhea.dat | 0 .../{custom => cmip5-custom}/CMOR_bdalb.dat | 0 .../{custom => cmip5-custom}/CMOR_bhalb.dat | 0 .../{custom => cmip5-custom}/CMOR_ch4s.dat | 0 .../{custom => cmip5-custom}/CMOR_chlora.dat | 0 .../CMOR_clhmtisccp.dat | 0 .../CMOR_clhtkisccp.dat | 0 .../{custom => cmip5-custom}/CMOR_clisccp.dat | 0 .../CMOR_cllmtisccp.dat | 0 .../CMOR_clltkisccp.dat | 0 .../CMOR_clmmtisccp.dat | 0 .../CMOR_clmtkisccp.dat | 0 .../CMOR_cltStderr.dat | 0 .../{custom => cmip5-custom}/CMOR_co2s.dat | 0 .../CMOR_coordinates.dat | 0 .../{custom => cmip5-custom}/CMOR_ctotal.dat | 0 .../{custom => cmip5-custom}/CMOR_dos.dat | 0 .../CMOR_dosStderr.dat | 0 .../{custom => cmip5-custom}/CMOR_dpn2o.dat | 0 .../{custom => cmip5-custom}/CMOR_et.dat | 0 .../CMOR_etStderr.dat | 0 .../{custom => cmip5-custom}/CMOR_fapar.dat | 0 .../CMOR_gppStderr.dat | 0 .../{custom => cmip5-custom}/CMOR_hfns.dat | 0 .../CMOR_hurStderr.dat | 0 .../CMOR_husStderr.dat | 0 .../CMOR_iwpStderr.dat | 0 .../CMOR_lapserate.dat | 0 .../{custom => cmip5-custom}/CMOR_lvp.dat | 0 .../{custom => cmip5-custom}/CMOR_lwcre.dat | 0 .../CMOR_lweGrace.dat | 0 .../{custom => cmip5-custom}/CMOR_lwp.dat | 0 .../CMOR_lwpStderr.dat | 0 .../{custom => cmip5-custom}/CMOR_n2oflux.dat | 0 .../{custom => cmip5-custom}/CMOR_n2os.dat | 0 .../{custom => cmip5-custom}/CMOR_netcre.dat | 0 .../CMOR_od550aerStderr.dat | 0 .../CMOR_od870aerStderr.dat | 0 .../{custom => cmip5-custom}/CMOR_ohc.dat | 0 .../CMOR_prStderr.dat | 0 .../{custom => cmip5-custom}/CMOR_prl.dat | 0 .../CMOR_prodlnox.dat | 0 .../{custom => cmip5-custom}/CMOR_ptype.dat | 0 .../{custom => cmip5-custom}/CMOR_qep.dat | 0 .../{custom => cmip5-custom}/CMOR_rlns.dat | 0 .../{custom => cmip5-custom}/CMOR_rlnst.dat | 0 .../{custom => cmip5-custom}/CMOR_rlnstcs.dat | 0 .../{custom => cmip5-custom}/CMOR_rlntcs.dat | 0 .../{custom => cmip5-custom}/CMOR_rluscs.dat | 0 .../{custom => cmip5-custom}/CMOR_rlut.dat | 0 .../{custom => cmip5-custom}/CMOR_rlutcs.dat | 0 .../{custom => cmip5-custom}/CMOR_rsns.dat | 0 .../{custom => cmip5-custom}/CMOR_rsnst.dat | 0 .../{custom => cmip5-custom}/CMOR_rsnstcs.dat | 0 .../CMOR_rsnstcsnorm.dat | 0 .../{custom => cmip5-custom}/CMOR_rsnt.dat | 0 .../{custom => cmip5-custom}/CMOR_rsntcs.dat | 0 .../{custom => cmip5-custom}/CMOR_rsut.dat | 0 .../{custom => cmip5-custom}/CMOR_rsutcs.dat | 0 .../{custom => cmip5-custom}/CMOR_rtnt.dat | 0 .../{custom => cmip5-custom}/CMOR_rx1day.dat | 0 .../{custom => cmip5-custom}/CMOR_rx5day.dat | 0 .../CMOR_siextent.dat | 0 .../{custom => cmip5-custom}/CMOR_sispeed.dat | 0 .../{custom => cmip5-custom}/CMOR_sithick.dat | 0 .../{custom => cmip5-custom}/CMOR_sm.dat | 0 .../{custom => cmip5-custom}/CMOR_sm1m.dat | 0 .../CMOR_smStderr.dat | 0 .../{custom => cmip5-custom}/CMOR_soz.dat | 0 .../{custom => cmip5-custom}/CMOR_swcre.dat | 0 .../CMOR_tasConf5.dat | 0 .../CMOR_tasConf95.dat | 0 .../{custom => cmip5-custom}/CMOR_tasa.dat | 0 .../{custom => cmip5-custom}/CMOR_tasaga.dat | 0 .../{custom => cmip5-custom}/CMOR_tcw.dat | 0 .../{custom => cmip5-custom}/CMOR_tnn.dat | 0 .../CMOR_tosStderr.dat | 0 .../{custom => cmip5-custom}/CMOR_toz.dat | 0 .../CMOR_tozStderr.dat | 0 .../CMOR_tro3prof.dat | 0 .../CMOR_tro3profStderr.dat | 0 .../{custom => cmip5-custom}/CMOR_troz.dat | 0 .../{custom => cmip5-custom}/CMOR_tsDay.dat | 0 .../{custom => cmip5-custom}/CMOR_tsLCDay.dat | 0 .../CMOR_tsLCNight.dat | 0 .../CMOR_tsLSSysErrDay.dat | 0 .../CMOR_tsLSSysErrNight.dat | 0 .../CMOR_tsLocalAtmErrDay.dat | 0 .../CMOR_tsLocalAtmErrNight.dat | 0 .../CMOR_tsLocalSfcErrDay.dat | 0 .../CMOR_tsLocalSfcErrNight.dat | 0 .../{custom => cmip5-custom}/CMOR_tsNight.dat | 0 .../CMOR_tsStderr.dat | 0 .../CMOR_tsTotalDay.dat | 0 .../CMOR_tsTotalNight.dat | 0 .../CMOR_tsUnCorErrDay.dat | 0 .../CMOR_tsUnCorErrNight.dat | 0 .../CMOR_tsVarDay.dat | 0 .../CMOR_tsVarNight.dat | 0 .../{custom => cmip5-custom}/CMOR_txx.dat | 0 .../{custom => cmip5-custom}/CMOR_uajet.dat | 0 .../{custom => cmip5-custom}/CMOR_vegfrac.dat | 0 .../{custom => cmip5-custom}/CMOR_xch4.dat | 0 .../{custom => cmip5-custom}/CMOR_xco2.dat | 0 .../tables/cmip6-custom/CMIP6_custom.json | 1649 +++++++++++++++++ .../cmip6-custom/convert-cmip5-to-cmip6.py | 60 + .../{Tables => tables}/CMIP7_aerosol.json | 0 .../cmip7/{Tables => tables}/CMIP7_atmos.json | 0 .../{Tables => tables}/CMIP7_atmosChem.json | 0 .../CMIP7_cell_measures.json | 0 .../{Tables => tables}/CMIP7_coordinate.json | 0 .../CMIP7_formula_terms.json | 0 .../cmip7/{Tables => tables}/CMIP7_grids.json | 0 .../cmip7/{Tables => tables}/CMIP7_land.json | 0 .../{Tables => tables}/CMIP7_landIce.json | 0 .../CMIP7_long_name_overrides.json | 0 .../cmip7/{Tables => tables}/CMIP7_ocean.json | 0 .../{Tables => tables}/CMIP7_ocnBgchem.json | 0 .../{Tables => tables}/CMIP7_seaIce.json | 0 esmvalcore/config/__init__.py | 14 +- esmvalcore/config/_config.py | 35 +- esmvalcore/config/_config_validators.py | 27 +- .../configurations/defaults/cmor_tables.yml | 77 + esmvalcore/dataset.py | 14 +- esmvalcore/io/__init__.py | 4 +- esmvalcore/io/local.py | 3 + .../cmor/_fixes/native6/test_era5.py | 6 - .../integration/cmor/test_read_cmor_tables.py | 105 +- tests/integration/cmor/test_table.py | 76 +- tests/integration/conftest.py | 101 +- tests/integration/io/test_local.py | 13 +- tests/integration/recipe/test_recipe.py | 10 +- tests/unit/config/test_config.py | 20 +- tests/unit/config/test_config_validator.py | 4 +- tests/unit/config/test_data_sources.py | 6 + tests/unit/io/local/test_get_data_sources.py | 14 +- tests/unit/preprocessor/test_configuration.py | 9 + 163 files changed, 2484 insertions(+), 361 deletions(-) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_MP_BC_tot.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_MP_CFCl3.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_MP_CH4.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_MP_CO.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_MP_CO2.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_MP_ClOX.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_MP_DU_tot.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_MP_N2O.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_MP_NH3.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_MP_NO.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_MP_NO2.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_MP_NOX.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_MP_O3.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_MP_OH.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_MP_S.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_MP_SO2.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_MP_SO4mm_tot.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_MP_SS_tot.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_agb.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_alb.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_albDiff.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_albDiffiTr13.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_amoc.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_asr.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_awhea.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_bdalb.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_bhalb.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_ch4s.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_chlora.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_clhmtisccp.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_clhtkisccp.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_clisccp.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_cllmtisccp.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_clltkisccp.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_clmmtisccp.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_clmtkisccp.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_cltStderr.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_co2s.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_coordinates.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_ctotal.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_dos.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_dosStderr.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_dpn2o.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_et.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_etStderr.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_fapar.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_gppStderr.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_hfns.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_hurStderr.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_husStderr.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_iwpStderr.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_lapserate.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_lvp.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_lwcre.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_lweGrace.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_lwp.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_lwpStderr.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_n2oflux.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_n2os.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_netcre.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_od550aerStderr.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_od870aerStderr.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_ohc.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_prStderr.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_prl.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_prodlnox.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_ptype.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_qep.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_rlns.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_rlnst.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_rlnstcs.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_rlntcs.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_rluscs.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_rlut.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_rlutcs.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_rsns.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_rsnst.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_rsnstcs.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_rsnstcsnorm.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_rsnt.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_rsntcs.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_rsut.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_rsutcs.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_rtnt.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_rx1day.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_rx5day.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_siextent.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_sispeed.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_sithick.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_sm.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_sm1m.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_smStderr.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_soz.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_swcre.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tasConf5.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tasConf95.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tasa.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tasaga.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tcw.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tnn.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tosStderr.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_toz.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tozStderr.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tro3prof.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tro3profStderr.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_troz.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tsDay.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tsLCDay.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tsLCNight.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tsLSSysErrDay.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tsLSSysErrNight.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tsLocalAtmErrDay.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tsLocalAtmErrNight.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tsLocalSfcErrDay.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tsLocalSfcErrNight.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tsNight.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tsStderr.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tsTotalDay.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tsTotalNight.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tsUnCorErrDay.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tsUnCorErrNight.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tsVarDay.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_tsVarNight.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_txx.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_uajet.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_vegfrac.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_xch4.dat (100%) rename esmvalcore/cmor/tables/{custom => cmip5-custom}/CMOR_xco2.dat (100%) create mode 100644 esmvalcore/cmor/tables/cmip6-custom/CMIP6_custom.json create mode 100644 esmvalcore/cmor/tables/cmip6-custom/convert-cmip5-to-cmip6.py rename esmvalcore/cmor/tables/cmip7/{Tables => tables}/CMIP7_aerosol.json (100%) rename esmvalcore/cmor/tables/cmip7/{Tables => tables}/CMIP7_atmos.json (100%) rename esmvalcore/cmor/tables/cmip7/{Tables => tables}/CMIP7_atmosChem.json (100%) rename esmvalcore/cmor/tables/cmip7/{Tables => tables}/CMIP7_cell_measures.json (100%) rename esmvalcore/cmor/tables/cmip7/{Tables => tables}/CMIP7_coordinate.json (100%) rename esmvalcore/cmor/tables/cmip7/{Tables => tables}/CMIP7_formula_terms.json (100%) rename esmvalcore/cmor/tables/cmip7/{Tables => tables}/CMIP7_grids.json (100%) rename esmvalcore/cmor/tables/cmip7/{Tables => tables}/CMIP7_land.json (100%) rename esmvalcore/cmor/tables/cmip7/{Tables => tables}/CMIP7_landIce.json (100%) rename esmvalcore/cmor/tables/cmip7/{Tables => tables}/CMIP7_long_name_overrides.json (100%) rename esmvalcore/cmor/tables/cmip7/{Tables => tables}/CMIP7_ocean.json (100%) rename esmvalcore/cmor/tables/cmip7/{Tables => tables}/CMIP7_ocnBgchem.json (100%) rename esmvalcore/cmor/tables/cmip7/{Tables => tables}/CMIP7_seaIce.json (100%) create mode 100644 esmvalcore/config/configurations/defaults/cmor_tables.yml diff --git a/doc/reference/cmor_tables.rst b/doc/reference/cmor_tables.rst index b969a938ab..b872817e13 100644 --- a/doc/reference/cmor_tables.rst +++ b/doc/reference/cmor_tables.rst @@ -49,10 +49,10 @@ by an underscore and the ``branding_suffix``. For example, the facets ``project: CMIP7, mip: atmos, short_name: tas, branding_suffix: tavg-h2m-hxy-u`` select one of the near-surface air temperature variables in the CMIP7 atmos table: -.. literalinclude:: ../../esmvalcore/cmor/tables/cmip7/Tables/CMIP7_atmos.json +.. literalinclude:: ../../esmvalcore/cmor/tables/cmip7/tables/CMIP7_atmos.json :start-at: "tas_tavg-h2m-hxy-u": { :end-at: }, - :caption: One of the ``tas`` variable definitions in the CMIP7 atmos table at `esmvalcore/cmor/tables/cmip7/Tables/CMIP7_atmos.json `__. + :caption: One of the ``tas`` variable definitions in the CMIP7 atmos table at `esmvalcore/cmor/tables/cmip7/tables/CMIP7_atmos.json `__. For other projects, the facet ``branding_suffix`` can also be used to distinguish between variables from the same CMOR table that share the same ``short_name``, diff --git a/esmvalcore/cmor/table.py b/esmvalcore/cmor/table.py index 71c519497b..f19978ce51 100644 --- a/esmvalcore/cmor/table.py +++ b/esmvalcore/cmor/table.py @@ -7,25 +7,30 @@ from __future__ import annotations import copy -import errno import glob +import importlib import json import logging import os from collections import Counter from functools import lru_cache, total_ordering from pathlib import Path -from typing import Union +from typing import TYPE_CHECKING import yaml from esmvalcore.exceptions import RecipeError -logger = logging.getLogger(__name__) +if TYPE_CHECKING: + from collections.abc import Iterable + from io import TextIOWrapper + + from esmvalcore.config import Config, Session + from esmvalcore.typing import Facets -CMORTable = Union["CMIP3Info", "CMIP5Info", "CMIP6Info", "CustomInfo"] +logger = logging.getLogger(__name__) -CMOR_TABLES: dict[str, CMORTable] = {} +CMOR_TABLES: dict[str, InfoBase] = {} """dict of str, obj: CMOR info objects.""" _CMOR_KEYS = ( @@ -37,18 +42,39 @@ ) -def _update_cmor_facets(facets): +def _get_institutes(project: str, dataset: str) -> list[str]: + """Return the institutes given the dataset name in CMIP6.""" + try: + return CMOR_TABLES[project].institutes[dataset] # type: ignore[attr-defined] + except (KeyError, AttributeError): + return [] + + +def _get_activity( + project: str, + exp: str | list[str], +) -> str | list[str] | None: + """Return the activity given the experiment name in CMIP6.""" + try: + if isinstance(exp, list): + return [CMOR_TABLES[project].activities[value][0] for value in exp] # type: ignore[attr-defined] + return CMOR_TABLES[project].activities[exp][0] # type: ignore[attr-defined] + except (KeyError, AttributeError): + return None + + +def _update_cmor_facets(facets: Facets) -> None: """Update `facets` with information from CMOR table.""" - project = facets["project"] - mip = facets["mip"] - short_name = facets["short_name"] - derive = facets.get("derive", False) + project: str = facets["project"] # type: ignore[assignment] + mip: str = facets["mip"] # type: ignore[assignment] + short_name: str = facets["short_name"] # type: ignore[assignment] + derive: bool = facets.get("derive", False) # type: ignore[assignment] table = CMOR_TABLES.get(project) if table: table_entry = table.get_variable( mip, short_name, - branding_suffix=facets.get("branding_suffix"), + branding_suffix=facets.get("branding_suffix"), # type: ignore[arg-type] derived=derive, ) else: @@ -71,6 +97,14 @@ def _update_cmor_facets(facets): key, facets, ) + if "dataset" in facets and "institute" not in facets: + institute = _get_institutes(project, facets["dataset"]) # type: ignore[arg-type] + if institute: + facets["institute"] = institute + if "exp" in facets and "activity" not in facets: + activity = _get_activity(project, facets["exp"]) # type: ignore[arg-type] + if activity: + facets["activity"] = activity def _get_mips(project: str, short_name: str) -> list[str]: @@ -154,6 +188,21 @@ def get_var_info( ) +def load_cmor_tables(cfg: Config) -> None: + """Load the configured CMOR tables into :data:`esmvalcore.cmor.table.CMOR_TABLES`. + + Parameters + ---------- + cfg: + The configuration. + """ + CMOR_TABLES.clear() + if cfg.get("config_developer_file") is not None: + read_cmor_tables(cfg["config_developer_file"]) + for project in cfg["projects"]: + CMOR_TABLES[project] = get_tables(cfg, project) + + def read_cmor_tables(cfg_developer: Path | None = None) -> None: """Read cmor tables required in the configuration. @@ -182,7 +231,7 @@ def read_cmor_tables(cfg_developer: Path | None = None) -> None: def _read_cmor_tables( cfg_file: Path, mtime: float, # noqa: ARG001 -) -> dict[str, CMORTable]: +) -> dict[str, InfoBase]: """Read cmor tables required in the configuration. Parameters @@ -201,7 +250,7 @@ def _read_cmor_tables( with open(var_alt_names_file, encoding="utf-8") as yfile: alt_names = yaml.safe_load(yfile) - cmor_tables: dict[str, CMORTable] = {} + cmor_tables: dict[str, InfoBase] = {} # Try to infer location for custom tables from config-developer.yml file, # if not possible, use default location @@ -264,6 +313,66 @@ def _read_table(cfg_developer, table, install_dir, custom, alt_names): raise ValueError(msg) +_TABLE_CACHE: dict[str, InfoBase] = {} +"""The CMOR tables are cached for faster access.""" + + +def clear_table_cache() -> None: + """Clear the CMOR table cache.""" + _TABLE_CACHE.clear() + + +def get_tables( + session: Session | Config, + project: str, +) -> InfoBase: + """Get the CMOR tables for a project. + + Parameters + ---------- + session: + The configuration. + project: + The project to load a CMOR table for. + """ + if project not in session["projects"]: + msg = f"Unknown project '{project}', please configure it under 'projects'." + raise ValueError(msg) + + kwargs = ( + session["projects"][project] + .get( + "cmor_table", + { + "type": "esmvalcore.cmor.table.NoInfo", + }, + ) + .copy() + ) + if "type" not in kwargs: + msg = ( + f"Missing CMOR table 'type' in configuration of project {project}. " + f"Current configuration is:\n{yaml.safe_dump(kwargs)}" + ) + raise ValueError(msg) + cache_key = str(kwargs) + if cache_key not in _TABLE_CACHE: + module_name, cls_name = kwargs.pop("type").rsplit(".", 1) + module = importlib.import_module(module_name) + cls = getattr(module, cls_name) + tables = cls(**kwargs) + if not isinstance(tables, InfoBase): + msg = ( + "Expected CMOR tables of type `esmvalcore.cmor.table.InfoBase`, " + f"but your configuration for project '{project}' contains " + f"'{tables}' of type '{type(tables)}'." + ) + raise TypeError(msg) + _TABLE_CACHE[cache_key] = tables + + return _TABLE_CACHE[cache_key] + + class InfoBase: """Base class for all table info classes. @@ -271,26 +380,63 @@ class InfoBase: Parameters ---------- - default: object - Default table to look variables on if not found + cmor_tables_path: + The path to a directory with subdirectory "Tables" where the CMOR tables + are located. + + default: + Default table to look variables on if not found. - alt_names: list[list[str]] - List of known alternative names for variables + alt_names: + List of known alternative names for variables. If no value is provided, + the default values from the file variable_alt_names.yml will be used. strict: bool If False, will look for a variable in other tables if it can not be - found in the requested one + found in the requested one. + + default_table_prefix: + If the table_id contains a prefix, it can be specified here. + + paths: + A list of paths to CMOR tables. If the path is relative and exists in + the ``tables`` directory in :mod:`esmvalcore.cmor`, the version of the + tables shipped with ESMValCore will be used. """ - def __init__(self, default, alt_names, strict): + def __init__( + self, + default: CustomInfo | None = None, + alt_names: list[list[str]] | None = None, + strict: bool = True, + paths: Iterable[Path] = (), + ) -> None: + # Configure the paths to the CMOR tables. + builtin_tables_path = Path(__file__).parent / "tables" + paths = tuple(Path(os.path.expandvars(p)).expanduser() for p in paths) + self.paths = tuple( + builtin_tables_path / p + if (builtin_tables_path / p).is_dir() + else p + for p in paths + ) + for path in self.paths: + if not path.is_dir(): + raise NotADirectoryError(path) + + # Configure the alternative names. if alt_names is None: - alt_names = "" - self.default = default + alt_names_path = Path(__file__).parent / "variable_alt_names.yml" + alt_names = yaml.safe_load( + alt_names_path.read_text(encoding="utf-8"), + ) self.alt_names = alt_names + self.coords: dict[str, CoordinateInfo] = {} + self.default = default self.strict = strict - self.tables = {} + self.tables: dict[str, TableInfo] = {} - def get_table(self, table): + def get_table(self, table: str) -> TableInfo | None: """Search and return the table info. Parameters @@ -365,15 +511,6 @@ def get_variable( # cmor_strict=False or derived=True var_info = self._look_in_all_tables(derived, alt_names_list) - # If that didn't work either, look in default table if - # cmor_strict=False or derived=True - if not var_info: - var_info = self._look_in_default( - derived, - alt_names_list, - table_name, - ) - # If necessary, adapt frequency of variable (set it to the one from the # requested MIP). E.g., if the user asked for table `Amon`, but the # variable has been found in `day`, use frequency `mon`. @@ -383,16 +520,6 @@ def get_variable( return var_info - def _look_in_default(self, derived, alt_names_list, table_name): - """Look for variable in default table.""" - var_info = None - if not self.strict or derived: - for alt_names in alt_names_list: - var_info = self.default.get_variable(table_name, alt_names) - if var_info: - break - return var_info - def _look_in_all_tables(self, derived, alt_names_list): """Look for variable in all tables.""" var_info = None @@ -439,53 +566,79 @@ class CMIP6Info(InfoBase): Parameters ---------- - cmor_tables_path: str - Path to the folder containing the Tables folder with the json files + cmor_tables_path: + The path to a directory with subdirectory "Tables" where the CMOR tables + are located. - default: object - Default table to look variables on if not found + default: + Default table to look variables on if not found. + + alt_names: + List of known alternative names for variables. If no value is provided, + the default values from the file variable_alt_names.yml will be used. strict: bool If False, will look for a variable in other tables if it can not be - found in the requested one + found in the requested one. + + default_table_prefix: + If the table_id contains a prefix, it can be specified here. + + paths: + A list of paths to CMOR tables. If the path is relative and exists in + the installed copy of the + `esmvalcore/cmor/tables `__ + directory it will be used. """ def __init__( self, - cmor_tables_path, - default=None, - alt_names=None, - strict=True, - default_table_prefix="", - ): - super().__init__(default, alt_names, strict) - cmor_tables_path = self._get_cmor_path(cmor_tables_path) - - self._cmor_folder = os.path.join(cmor_tables_path, "Tables") - if glob.glob(os.path.join(self._cmor_folder, "*_CV.json")): - self._load_controlled_vocabulary() + cmor_tables_path: str | None = None, + default: CustomInfo | None = None, + alt_names: list[list[str]] | None = None, + strict: bool = True, + default_table_prefix: str = "", + paths: Iterable[Path] = (), + ) -> None: + if cmor_tables_path is not None: + # Support cmor_tables_path for backward compatibility. + # TODO: remove in v2.16.0 + tables_path = Path(self._get_cmor_path(cmor_tables_path)) + if (tables_path / "tables").exists(): + # Support CMIP7 which uses a lowercase "tables" subdirectory. + cmor_folder = tables_path / "tables" + else: + cmor_folder = tables_path / "Tables" + paths = (*tuple(paths), cmor_folder) + super().__init__(default, alt_names, strict, paths=paths) self.default_table_prefix = default_table_prefix + self.var_to_freq: dict[str, dict[str, str]] = {} + self.activities: dict[str, list[str]] = {} + self.institutes: dict[str, list[str]] = {} - self.var_to_freq = {} - - self._load_coordinates() - for json_file in glob.glob(os.path.join(self._cmor_folder, "*.json")): - if "CV_test" in json_file or "grids" in json_file: - continue - try: - self._load_table(json_file) - except Exception: - msg = f"Exception raised when loading {json_file}" - # Logger may not be ready at this stage - if logger.handlers: - logger.error(msg) - else: - print(msg) # noqa: T201 - raise + for path in self.paths: + if not any(path.glob("*.json")): + msg = f"No CMOR tables found in {path}" + raise ValueError(msg) + self._load_controlled_vocabulary(path) + self._load_coordinates(path) + for json_file in glob.glob(os.path.join(path, "*.json")): + if "CV_test" in json_file or "grids" in json_file: + continue + try: + self._load_table(json_file) + except Exception: + msg = f"Exception raised when loading {json_file}" + # Logger may not be ready at this stage + if logger.handlers: + logger.error(msg) + else: + print(msg) # noqa: T201 + raise @staticmethod - def _get_cmor_path(cmor_tables_path): + def _get_cmor_path(cmor_tables_path: str) -> str: if os.path.isdir(cmor_tables_path): return cmor_tables_path cwd = os.path.dirname(os.path.realpath(__file__)) @@ -500,13 +653,15 @@ def _load_table(self, json_file): raw_data = json.loads(inf.read()) if not self._is_table(raw_data): return - table = TableInfo() header = raw_data["Header"] - table.name = header["table_id"].split(" ")[-1] - self.tables[table.name] = table + table_name = header["table_id"].split(" ")[-1] + if table_name not in self.tables: + table = TableInfo() + table.name = table_name + self.tables[table_name] = table + table = self.tables[table_name] generic_levels = header["generic_levels"].split() - table.frequency = header.get("frequency", "") self.var_to_freq[table.name] = {} for var_name, var_data in raw_data["variable_entry"].items(): @@ -520,7 +675,6 @@ def _load_table(self, json_file): var_freqs = (var.frequency for var in table.values()) table_freq, _ = Counter(var_freqs).most_common(1)[0] table.frequency = table_freq - self.tables[table.name] = table def _assign_dimensions(self, var, generic_levels): for dimension in var.dimensions: @@ -544,10 +698,9 @@ def _assign_dimensions(self, var, generic_levels): var.coordinates[dimension] = coord - def _load_coordinates(self): - self.coords = {} + def _load_coordinates(self, path: Path) -> None: for json_file in glob.glob( - os.path.join(self._cmor_folder, "*coordinate*.json"), + os.path.join(path, "*coordinate*.json"), ): with open(json_file, encoding="utf-8") as inf: table_data = json.loads(inf.read()) @@ -556,11 +709,9 @@ def _load_coordinates(self): coord.read_json(table_data["axis_entry"][coord_name]) self.coords[coord_name] = coord - def _load_controlled_vocabulary(self): - self.activities = {} - self.institutes = {} + def _load_controlled_vocabulary(self, path: Path) -> None: for json_file in glob.glob( - os.path.join(self._cmor_folder, "*_CV.json"), + os.path.join(path, "*_CV.json"), ): with open(json_file, encoding="utf-8") as inf: table_data = json.loads(inf.read()) @@ -606,6 +757,58 @@ def _is_table(table_data): return "Header" in table_data +class Obs4MIPsInfo(CMIP6Info): + """Class to read obs4MIPs-like data request. + + This uses CMOR 3 json format + + Parameters + ---------- + cmor_tables_path: + The path to a directory with subdirectory "Tables" where the CMOR tables + are located. + + default: + Default table to look variables on if not found. + + alt_names: + List of known alternative names for variables. If no value is provided, + the default values from the file variable_alt_names.yml will be used. + + strict: bool + If False, will look for a variable in other tables if it can not be + found in the requested one. + + paths: + A list of paths to CMOR tables. If the path is relative and exists in + the installed copy of the + `esmvalcore/cmor/tables `__ + directory it will be used. + """ + + def __init__( + self, + cmor_tables_path: str | None = None, + default: CustomInfo | None = None, + alt_names: list[list[str]] | None = None, + strict: bool = True, + paths: Iterable[Path] = (), + ) -> None: + super().__init__( + cmor_tables_path=cmor_tables_path, + default=default, + alt_names=alt_names, + strict=strict, + paths=paths, + ) + # Remove the prefix from the table_id. + table_id_prefix = "obs4MIPs_" + for name in list(self.tables): + if name.startswith(table_id_prefix): + table = self.tables.pop(name) + self.tables[name[len(table_id_prefix) :]] = table + + @total_ordering class TableInfo(dict): """Container class for storing a CMOR table.""" @@ -698,6 +901,7 @@ def __init__(self, table_type, short_name): Variable's short name. """ super().__init__() + self.name = short_name self.table_type = table_type self.modeling_realm = [] """Modeling realm""" @@ -729,6 +933,9 @@ def __init__(self, table_type, short_name): self._json_data = None + def __repr__(self) -> str: + return f"{self.__class__.__name__}(name={self.name})" + def copy(self): """Return a shallow copy of VariableInfo. @@ -881,54 +1088,66 @@ class CMIP5Info(InfoBase): Parameters ---------- - cmor_tables_path: str - Path to the folder containing the Tables folder with the json files + cmor_tables_path: + The path to a directory with subdirectory "Tables" where the CMOR tables + are located. - default: object - Default table to look variables on if not found + default: + Default table to look variables on if not found. + + alt_names: + List of known alternative names for variables. If no value is provided, + the default values from the file variable_alt_names.yml will be used. strict: bool If False, will look for a variable in other tables if it can not be - found in the requested one + found in the requested one. + + paths: + A list of paths to CMOR tables. If the path is relative and exists in + the installed copy of the + `esmvalcore/cmor/tables `__ + directory it will be used. """ def __init__( self, - cmor_tables_path, - default=None, - alt_names=None, - strict=True, - ): - super().__init__(default, alt_names, strict) - cmor_tables_path = self._get_cmor_path(cmor_tables_path) - - self._cmor_folder = os.path.join(cmor_tables_path, "Tables") - if not os.path.isdir(self._cmor_folder): - raise OSError( - errno.ENOTDIR, - "CMOR tables path is not a directory", - self._cmor_folder, - ) - - self.strict = strict - self.tables = {} - self.coords = {} - self._current_table = None - self._last_line_read = None - - for table_file in glob.glob(os.path.join(self._cmor_folder, "*")): - if "_grids" in table_file: - continue - try: - self._load_table(table_file) - except Exception: - msg = f"Exception raised when loading {table_file}" - # Logger may not be ready at this stage - if logger.handlers: - logger.error(msg) - else: - print(msg) # noqa: T201 - raise + cmor_tables_path: str | None = None, + default: CustomInfo | None = None, + alt_names: list[list[str]] | None = None, + strict: bool = True, + paths: Iterable[Path] = (), + ) -> None: + if cmor_tables_path is not None: + # Support cmor_tables_path for backward compatibility. + # TODO: remove in v2.16.0 + cmor_tables_path = self._get_cmor_path(cmor_tables_path) + cmor_folder = Path(cmor_tables_path) / "Tables" + paths = (*tuple(paths), cmor_folder) + super().__init__(default, alt_names, strict, paths=paths) + + self._current_table: TextIOWrapper | None = None + self._last_line_read = ("", "") + + for path in self.paths: + for table_file in sorted( + glob.glob(os.path.join(path, "*")), + # Read coordinate files before variable files so we can link the + # variables with the coordinates. + key=lambda filename: "coordinate" not in filename, + ): + if "_grids" in table_file: + continue + try: + self._load_table(table_file) + except Exception: + msg = f"Exception raised when loading {table_file}" + # Logger may not be ready at this stage + if logger.handlers: + logger.error(msg) + else: + print(msg) # noqa: T201 + raise @staticmethod def _get_cmor_path(cmor_tables_path): @@ -937,25 +1156,21 @@ def _get_cmor_path(cmor_tables_path): cwd = os.path.dirname(os.path.realpath(__file__)) return os.path.join(cwd, "tables", cmor_tables_path) - def _load_table(self, table_file, table_name=""): - if table_name and table_name in self.tables: - # special case used for updating a table with custom variable file - table = self.tables[table_name] + def _load_table(self, table_file: str) -> None: + table = self._read_table_file(table_file) + if table.name in self.tables: + self.tables[table.name].update(table) else: - # default case: table name is first line of table file - table = None - - self._read_table_file(table_file, table) + self.tables[table.name] = table - def _read_table_file(self, table_file, table=None): + def _read_table_file(self, table_file: str) -> TableInfo: + table = TableInfo() with open(table_file, encoding="utf-8") as self._current_table: self._read_line() while True: key, value = self._last_line_read if key == "table_id": - table = TableInfo() table.name = value[len("Table ") :] - self.tables[table.name] = table elif key == "frequency": table.frequency = value elif key == "modeling_realm": @@ -973,7 +1188,8 @@ def _read_table_file(self, table_file, table=None): table[value] = self._read_variable(value, table.frequency) continue if not self._read_line(): - return + break + return table def _read_line(self): line = self._current_table.readline() @@ -1047,24 +1263,35 @@ class CMIP3Info(CMIP5Info): Parameters ---------- - cmor_tables_path: str - Path to the folder containing the Tables folder with the json files + cmor_tables_path: + The path to a directory with subdirectory "Tables" where the CMOR tables + are located. + + default: + Default table to look variables on if not found. - default: object - Default table to look variables on if not found + alt_names: + List of known alternative names for variables. If no value is provided, + the default values from the file variable_alt_names.yml will be used. strict: bool If False, will look for a variable in other tables if it can not be - found in the requested one + found in the requested one. + + paths: + A list of paths to CMOR tables. If the path is relative and exists in + the installed copy of the + `esmvalcore/cmor/tables `__ + directory it will be used. """ - def _read_table_file(self, table_file, table=None): + def _read_table_file(self, table_file: str) -> TableInfo: for dim in ("zlevel",): coord = CoordinateInfo(dim) coord.generic_level = True coord.axis = "Z" self.coords[dim] = coord - super()._read_table_file(table_file, table) + return super()._read_table_file(table_file) def _read_coordinate(self, value): coord = super()._read_coordinate(value) @@ -1075,8 +1302,8 @@ def _read_coordinate(self, value): def _read_variable(self, short_name, frequency): var = super()._read_variable(short_name, frequency) - var.frequency = None - var.modeling_realm = None + var.frequency = "" + var.modeling_realm = [] return var @@ -1096,28 +1323,28 @@ def __init__(self, cmor_tables_path: str | Path | None = None) -> None: """Initialize class member.""" self.coords = {} self.tables = {} - self.var_to_freq: dict[str, dict] = {} + self.var_to_freq: dict[str, dict[str, str]] = {} table = TableInfo() table.name = "custom" self.tables[table.name] = table # First, read default custom tables from repository - self._cmor_folder = self._get_cmor_path("custom") - self._read_table_dir(self._cmor_folder) + self.paths = (Path(self._get_cmor_path("cmip5-custom")),) # Second, if given, update default tables with user-defined custom # tables if cmor_tables_path is not None: - self._user_table_folder = self._get_cmor_path(cmor_tables_path) - if not os.path.isdir(self._user_table_folder): + user_table_folder = Path(self._get_cmor_path(cmor_tables_path)) + if not user_table_folder.is_dir(): msg = ( - f"Custom CMOR tables path {self._user_table_folder} is " + f"Custom CMOR tables path {user_table_folder} is " f"not a directory" ) raise ValueError(msg) - self._read_table_dir(self._user_table_folder) - else: - self._user_table_folder = None + self.paths += (user_table_folder,) + + for path in self.paths: + self._read_table_dir(str(path)) def _read_table_dir(self, table_dir: str) -> None: """Read CMOR tables from directory.""" @@ -1131,7 +1358,7 @@ def _read_table_dir(self, table_dir: str) -> None: if dat_file == coordinates_file: continue try: - self._read_table_file(dat_file) + self._load_table(dat_file) except Exception: msg = f"Exception raised when loading {dat_file}" # Logger may not be ready at this stage @@ -1174,12 +1401,10 @@ def get_variable( """ return self.tables["custom"].get(short_name, None) - def _read_table_file( - self, - table_file: str, - table: TableInfo | None = None, # noqa: ARG002 - ) -> None: + def _read_table_file(self, table_file: str) -> TableInfo: """Read a single table file.""" + table = TableInfo() + table.name = "custom" with open(table_file, encoding="utf-8") as self._current_table: self._read_line() while True: @@ -1194,13 +1419,44 @@ def _read_table_file( self.coords[value] = self._read_coordinate(value) continue elif key == "variable_entry": - self.tables["custom"][value] = self._read_variable( - value, - "", - ) + table[value] = self._read_variable(value, "") continue if not self._read_line(): - return + return table + + +class NoInfo(InfoBase): + """Table that can be used for projects that do not have a CMOR table.""" + + def get_variable( + self, + table_name: str, # noqa: ARG002 + short_name: str, + *, + branding_suffix: str | None = None, # noqa: ARG002 + derived: bool = False, # noqa: ARG002 + ) -> VariableInfo | None: + """Search and return the variable information. + + Parameters + ---------- + table_name: + Table name, i.e., the variable's MIP. + short_name: + Variable's short name. + derived: + Variable is derived. Information retrieval for derived variables + always looks in the default tables (usually, the custom tables) if + variable is not found in the requested table. + + Returns + ------- + VariableInfo | None + `VariableInfo` object for the requested variable if found, ``None`` + otherwise. + + """ + return VariableInfo(table_type="No table", short_name=short_name) # Load the default tables on initializing the module. diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_BC_tot.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_BC_tot.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_BC_tot.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_BC_tot.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_CFCl3.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_CFCl3.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_CFCl3.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_CFCl3.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_CH4.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_CH4.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_CH4.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_CH4.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_CO.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_CO.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_CO.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_CO.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_CO2.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_CO2.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_CO2.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_CO2.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_ClOX.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_ClOX.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_ClOX.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_ClOX.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_DU_tot.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_DU_tot.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_DU_tot.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_DU_tot.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_N2O.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_N2O.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_N2O.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_N2O.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_NH3.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_NH3.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_NH3.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_NH3.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_NO.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_NO.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_NO.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_NO.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_NO2.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_NO2.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_NO2.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_NO2.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_NOX.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_NOX.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_NOX.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_NOX.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_O3.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_O3.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_O3.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_O3.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_OH.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_OH.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_OH.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_OH.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_S.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_S.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_S.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_S.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_SO2.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_SO2.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_SO2.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_SO2.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_SO4mm_tot.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_SO4mm_tot.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_SO4mm_tot.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_SO4mm_tot.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_SS_tot.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_SS_tot.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_SS_tot.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_SS_tot.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_agb.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_agb.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_agb.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_agb.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_alb.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_alb.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_alb.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_alb.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_albDiff.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_albDiff.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_albDiff.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_albDiff.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_albDiffiTr13.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_albDiffiTr13.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_albDiffiTr13.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_albDiffiTr13.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_amoc.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_amoc.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_amoc.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_amoc.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_asr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_asr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_asr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_asr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_awhea.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_awhea.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_awhea.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_awhea.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_bdalb.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_bdalb.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_bdalb.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_bdalb.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_bhalb.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_bhalb.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_bhalb.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_bhalb.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_ch4s.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_ch4s.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_ch4s.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_ch4s.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_chlora.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_chlora.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_chlora.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_chlora.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_clhmtisccp.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_clhmtisccp.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_clhmtisccp.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_clhmtisccp.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_clhtkisccp.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_clhtkisccp.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_clhtkisccp.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_clhtkisccp.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_clisccp.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_clisccp.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_clisccp.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_clisccp.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_cllmtisccp.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_cllmtisccp.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_cllmtisccp.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_cllmtisccp.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_clltkisccp.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_clltkisccp.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_clltkisccp.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_clltkisccp.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_clmmtisccp.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_clmmtisccp.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_clmmtisccp.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_clmmtisccp.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_clmtkisccp.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_clmtkisccp.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_clmtkisccp.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_clmtkisccp.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_cltStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_cltStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_cltStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_cltStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_co2s.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_co2s.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_co2s.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_co2s.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_coordinates.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_coordinates.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_coordinates.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_coordinates.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_ctotal.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_ctotal.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_ctotal.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_ctotal.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_dos.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_dos.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_dos.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_dos.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_dosStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_dosStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_dosStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_dosStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_dpn2o.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_dpn2o.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_dpn2o.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_dpn2o.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_et.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_et.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_et.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_et.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_etStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_etStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_etStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_etStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_fapar.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_fapar.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_fapar.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_fapar.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_gppStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_gppStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_gppStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_gppStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_hfns.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_hfns.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_hfns.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_hfns.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_hurStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_hurStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_hurStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_hurStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_husStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_husStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_husStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_husStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_iwpStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_iwpStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_iwpStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_iwpStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_lapserate.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_lapserate.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_lapserate.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_lapserate.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_lvp.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_lvp.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_lvp.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_lvp.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_lwcre.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_lwcre.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_lwcre.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_lwcre.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_lweGrace.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_lweGrace.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_lweGrace.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_lweGrace.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_lwp.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_lwp.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_lwp.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_lwp.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_lwpStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_lwpStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_lwpStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_lwpStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_n2oflux.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_n2oflux.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_n2oflux.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_n2oflux.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_n2os.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_n2os.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_n2os.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_n2os.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_netcre.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_netcre.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_netcre.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_netcre.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_od550aerStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_od550aerStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_od550aerStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_od550aerStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_od870aerStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_od870aerStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_od870aerStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_od870aerStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_ohc.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_ohc.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_ohc.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_ohc.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_prStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_prStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_prStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_prStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_prl.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_prl.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_prl.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_prl.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_prodlnox.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_prodlnox.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_prodlnox.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_prodlnox.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_ptype.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_ptype.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_ptype.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_ptype.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_qep.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_qep.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_qep.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_qep.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rlns.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rlns.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rlns.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rlns.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rlnst.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rlnst.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rlnst.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rlnst.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rlnstcs.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rlnstcs.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rlnstcs.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rlnstcs.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rlntcs.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rlntcs.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rlntcs.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rlntcs.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rluscs.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rluscs.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rluscs.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rluscs.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rlut.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rlut.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rlut.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rlut.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rlutcs.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rlutcs.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rlutcs.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rlutcs.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rsns.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rsns.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rsns.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rsns.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rsnst.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rsnst.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rsnst.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rsnst.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rsnstcs.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rsnstcs.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rsnstcs.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rsnstcs.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rsnstcsnorm.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rsnstcsnorm.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rsnstcsnorm.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rsnstcsnorm.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rsnt.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rsnt.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rsnt.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rsnt.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rsntcs.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rsntcs.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rsntcs.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rsntcs.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rsut.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rsut.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rsut.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rsut.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rsutcs.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rsutcs.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rsutcs.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rsutcs.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rtnt.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rtnt.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rtnt.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rtnt.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rx1day.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rx1day.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rx1day.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rx1day.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rx5day.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rx5day.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rx5day.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rx5day.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_siextent.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_siextent.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_siextent.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_siextent.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_sispeed.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_sispeed.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_sispeed.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_sispeed.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_sithick.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_sithick.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_sithick.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_sithick.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_sm.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_sm.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_sm.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_sm.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_sm1m.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_sm1m.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_sm1m.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_sm1m.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_smStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_smStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_smStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_smStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_soz.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_soz.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_soz.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_soz.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_swcre.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_swcre.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_swcre.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_swcre.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tasConf5.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tasConf5.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tasConf5.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tasConf5.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tasConf95.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tasConf95.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tasConf95.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tasConf95.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tasa.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tasa.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tasa.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tasa.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tasaga.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tasaga.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tasaga.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tasaga.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tcw.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tcw.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tcw.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tcw.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tnn.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tnn.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tnn.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tnn.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tosStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tosStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tosStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tosStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_toz.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_toz.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_toz.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_toz.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tozStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tozStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tozStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tozStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tro3prof.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tro3prof.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tro3prof.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tro3prof.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tro3profStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tro3profStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tro3profStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tro3profStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_troz.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_troz.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_troz.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_troz.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsDay.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsDay.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsDay.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsDay.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsLCDay.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLCDay.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsLCDay.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLCDay.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsLCNight.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLCNight.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsLCNight.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLCNight.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsLSSysErrDay.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLSSysErrDay.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsLSSysErrDay.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLSSysErrDay.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsLSSysErrNight.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLSSysErrNight.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsLSSysErrNight.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLSSysErrNight.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsLocalAtmErrDay.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLocalAtmErrDay.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsLocalAtmErrDay.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLocalAtmErrDay.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsLocalAtmErrNight.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLocalAtmErrNight.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsLocalAtmErrNight.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLocalAtmErrNight.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsLocalSfcErrDay.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLocalSfcErrDay.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsLocalSfcErrDay.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLocalSfcErrDay.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsLocalSfcErrNight.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLocalSfcErrNight.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsLocalSfcErrNight.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLocalSfcErrNight.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsNight.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsNight.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsNight.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsNight.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsTotalDay.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsTotalDay.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsTotalDay.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsTotalDay.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsTotalNight.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsTotalNight.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsTotalNight.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsTotalNight.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsUnCorErrDay.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsUnCorErrDay.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsUnCorErrDay.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsUnCorErrDay.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsUnCorErrNight.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsUnCorErrNight.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsUnCorErrNight.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsUnCorErrNight.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsVarDay.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsVarDay.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsVarDay.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsVarDay.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsVarNight.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsVarNight.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsVarNight.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsVarNight.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_txx.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_txx.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_txx.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_txx.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_uajet.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_uajet.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_uajet.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_uajet.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_vegfrac.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_vegfrac.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_vegfrac.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_vegfrac.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_xch4.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_xch4.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_xch4.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_xch4.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_xco2.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_xco2.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_xco2.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_xco2.dat diff --git a/esmvalcore/cmor/tables/cmip6-custom/CMIP6_custom.json b/esmvalcore/cmor/tables/cmip6-custom/CMIP6_custom.json new file mode 100644 index 0000000000..7628aa821e --- /dev/null +++ b/esmvalcore/cmor/tables/cmip6-custom/CMIP6_custom.json @@ -0,0 +1,1649 @@ +{ + "Header": { + "generic_levels": "olevel", + "table_id": "Table custom" + }, + "variable_entry": { + "MP_BC_tot": { + "comment": "positive mass of BC_tot in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of black carbon (sum of all aerosol modes)", + "modeling_realm": "atmos", + "out_name": "MP_BC_tot", + "standard_name": "", + "units": "kg" + }, + "MP_CFCl3": { + "comment": "positive mass of CFCl3", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of CFCl3 (CFC-11)", + "modeling_realm": "atmos", + "out_name": "MP_CFCl3", + "standard_name": "", + "units": "kg" + }, + "MP_CH4": { + "comment": "positive mass of CH4 in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of CH4", + "modeling_realm": "atmos", + "out_name": "MP_CH4", + "standard_name": "", + "units": "kg" + }, + "MP_CO": { + "comment": "positive mass of CO in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of CO", + "modeling_realm": "atmos", + "out_name": "MP_CO", + "standard_name": "", + "units": "kg" + }, + "MP_CO2": { + "comment": "positive mass of CO2 in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of CO2", + "modeling_realm": "atmos", + "out_name": "MP_CO2", + "standard_name": "", + "units": "kg" + }, + "MP_ClOX": { + "comment": "positive mass of ClOX", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of ClOX", + "modeling_realm": "atmos", + "out_name": "MP_ClOX", + "standard_name": "", + "units": "kg" + }, + "MP_DU_tot": { + "comment": "positive mass of DU_tot in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of mineral dust (sum of all aerosol modes)", + "modeling_realm": "atmos", + "out_name": "MP_DU_tot", + "standard_name": "", + "units": "kg" + }, + "MP_N2O": { + "comment": "positive mass of N2O in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of N2O", + "modeling_realm": "atmos", + "out_name": "MP_N2O", + "standard_name": "", + "units": "kg" + }, + "MP_NH3": { + "comment": "positive mass of NH3 in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of NH3", + "modeling_realm": "atmos", + "out_name": "MP_NH3", + "standard_name": "", + "units": "kg" + }, + "MP_NO": { + "comment": "positive mass of NO in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of NO", + "modeling_realm": "atmos", + "out_name": "MP_NO", + "standard_name": "", + "units": "kg" + }, + "MP_NO2": { + "comment": "positive mass of NO2 in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of NO2", + "modeling_realm": "atmos", + "out_name": "MP_NO2", + "standard_name": "", + "units": "kg" + }, + "MP_NOX": { + "comment": "positive mass of NOX in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of NOX (NO+NO2)", + "modeling_realm": "atmos", + "out_name": "MP_NOX", + "standard_name": "", + "units": "kg" + }, + "MP_O3": { + "comment": "positive mass of O3 in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of O3", + "modeling_realm": "atmos", + "out_name": "MP_O3", + "standard_name": "", + "units": "kg" + }, + "MP_OH": { + "comment": "positive mass of OH in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of OH", + "modeling_realm": "atmos", + "out_name": "MP_OH", + "standard_name": "", + "units": "kg" + }, + "MP_S": { + "comment": "positive mass of S in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of S", + "modeling_realm": "atmos", + "out_name": "MP_S", + "standard_name": "", + "units": "kg" + }, + "MP_SO2": { + "comment": "positive mass of SO2 in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of SO2", + "modeling_realm": "atmos", + "out_name": "MP_SO2", + "standard_name": "", + "units": "kg" + }, + "MP_SO4mm_tot": { + "comment": "positive mass of SO4mm_tot in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of aerosol sulfate (sum of all aerosol modes)", + "modeling_realm": "atmos", + "out_name": "MP_SO4mm_tot", + "standard_name": "", + "units": "kg" + }, + "MP_SS_tot": { + "comment": "positive mass of SS_tot in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of sea salt (sum of all aerosol modes)", + "modeling_realm": "atmos", + "out_name": "MP_SS_tot", + "standard_name": "", + "units": "kg" + }, + "agb": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "Amount of living biomass (organic matter) stored in vegetation above the soil including stem, stump, branches, bark, seeds and foliage, expressed as dry weight", + "dimensions": "longitude latitude time", + "long_name": "Above-Ground Biomass", + "modeling_realm": "land", + "out_name": "agb", + "standard_name": "", + "type": "real", + "units": "kg m-2", + "valid_min": "0.0" + }, + "alb": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "calculated from clear-sky fluxes", + "dimensions": "longitude latitude time", + "long_name": "albedo at the surface", + "modeling_realm": "atmos", + "out_name": "alb", + "standard_name": "", + "type": "real", + "units": "1" + }, + "albDiff": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude time", + "long_name": "Difference in surface albedo for a given vegetation cover transition", + "modeling_realm": "atmos", + "out_name": "albDiff", + "standard_name": "", + "type": "real", + "units": "1" + }, + "albDiffiTr13": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude time", + "long_name": "Difference in Surface Albedo for Vegetation Cover Transition from Forest to Crops and Grasses", + "modeling_realm": "atmos", + "out_name": "albDiffiTr13", + "standard_name": "", + "type": "real", + "units": "1" + }, + "amoc": { + "cell_measures": "area: areacello", + "cell_methods": "time: mean area: where sea", + "comment": "AMOC at the Rapid array (26.5 N)", + "dimensions": "time", + "long_name": "Atlantic Meridional Overturning Circulation", + "modeling_realm": "ocean", + "out_name": "amoc", + "standard_name": "", + "type": "real", + "units": "kg s-1" + }, + "asr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude time", + "long_name": "Absorbed shortwave radiation", + "modeling_realm": "atmos", + "out_name": "asr", + "positive": "down", + "standard_name": "", + "type": "real", + "units": "W m-2" + }, + "awhea": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "global mean net surface heat flux over open water", + "dimensions": "longitude latitude time", + "long_name": "Global Mean Net Surface Heat Flux Over Open Water", + "modeling_realm": "atmos", + "out_name": "awhea", + "standard_name": "", + "type": "real", + "units": "W m-2" + }, + "bdalb": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude time", + "long_name": "Broadband directional albedo at the surface", + "modeling_realm": "land", + "out_name": "bdalb", + "standard_name": "", + "type": "real", + "units": "1" + }, + "bhalb": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude time", + "long_name": "Broadband bihemispherical albedo at the surface", + "modeling_realm": "land", + "out_name": "bhalb", + "standard_name": "", + "type": "real", + "units": "1" + }, + "ch4s": { + "cell_measures": "area: areacella", + "cell_methods": "area: time: mean", + "comment": "As ch4, but only at the surface", + "dimensions": "longitude latitude time", + "long_name": "Atmosphere CH4 surface", + "modeling_realm": "atmos", + "out_name": "ch4s", + "standard_name": "mole_fraction_of_methane_in_air", + "type": "real", + "units": "1e-09" + }, + "chlora": { + "cell_measures": "area: areacello", + "cell_methods": "area: mean where sea time: mean", + "comment": "calculated at the surface (to be compared with satellite measurements)", + "dimensions": "longitude latitude time", + "long_name": "chlorophyll concentration", + "modeling_realm": "ocean", + "out_name": "chlora", + "standard_name": "", + "type": "real", + "units": "kg m-3" + }, + "clhmtisccp": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at the top of the atmosphere (to be compared with satellite measurements)", + "dimensions": "longitude latitude time", + "long_name": "ISCCP High Level Medium-Thickness Cloud Area Fraction", + "modeling_realm": "atmos", + "out_name": "clhmtisccp", + "standard_name": "", + "type": "real", + "units": "%" + }, + "clhtkisccp": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at the top of the atmosphere (to be compared with satellite measurements)", + "dimensions": "longitude latitude time", + "long_name": "ISCCP high level thick cloud area fraction", + "modeling_realm": "atmos", + "out_name": "clhtkisccp", + "standard_name": "", + "type": "real", + "units": "%" + }, + "clisccp": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude plev19 tau time", + "long_name": "ISCCP Cloud Area Fraction", + "modeling_realm": "atmos", + "out_name": "clisccp", + "standard_name": "", + "type": "real", + "units": "%" + }, + "cllmtisccp": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at the top of the atmosphere (to be compared with satellite measurements)", + "dimensions": "longitude latitude time", + "long_name": "ISCCP Low Level Medium-Thickness Cloud Area Fraction", + "modeling_realm": "atmos", + "out_name": "cllmtisccp", + "standard_name": "", + "type": "real", + "units": "%" + }, + "clltkisccp": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at the top of the atmosphere (to be compared with satellite measurements)", + "dimensions": "longitude latitude time", + "long_name": "ISCCP low level thick cloud area fraction", + "modeling_realm": "atmos", + "out_name": "clltkisccp", + "standard_name": "", + "type": "real", + "units": "%" + }, + "clmmtisccp": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at the top of the atmosphere (to be compared with satellite measurements)", + "dimensions": "longitude latitude time", + "long_name": "ISCCP Middle Level Medium-Thickness Cloud Area Fraction", + "modeling_realm": "atmos", + "out_name": "clmmtisccp", + "standard_name": "", + "type": "real", + "units": "%" + }, + "clmtkisccp": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at the top of the atmosphere (to be compared with satellite measurements)", + "dimensions": "longitude latitude time", + "long_name": "ISCCP Middle Level Thick Cloud Area Fraction", + "modeling_realm": "atmos", + "out_name": "clmtkisccp", + "standard_name": "", + "type": "real", + "units": "%" + }, + "cltStderr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "for the whole atmospheric column, as seen from the surface or the top of the atmosphere. Include both large-scale and convective cloud.", + "dimensions": "longitude latitude time", + "long_name": "Total Cloud Fraction Error", + "modeling_realm": "atmos", + "ok_max_mean_abs": "0.01", + "ok_min_mean_abs": "0", + "out_name": "cltStderr", + "standard_name": "", + "type": "real", + "units": "%", + "valid_max": "0.01", + "valid_min": "0" + }, + "co2s": { + "cell_measures": "area: areacella", + "cell_methods": "area: time: mean", + "comment": "As co2, but only at the surface", + "dimensions": "longitude latitude time", + "long_name": "Atmosphere CO2", + "modeling_realm": "atmos", + "out_name": "co2s", + "standard_name": "mole_fraction_of_carbon_dioxide_in_air", + "type": "real", + "units": "1e-06" + }, + "ctotal": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean area: mean", + "dimensions": "longitude latitude time", + "long_name": "Total Carbon Mass in Ecosystem", + "modeling_realm": "land", + "out_name": "ctotal", + "standard_name": "", + "type": "real", + "units": "kg m-2" + }, + "dos": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean area: mean where land", + "comment": "(unitless) degree of soil saturation for comparing mass based models with volumetric observations.", + "dimensions": "longitude latitude time", + "long_name": "Degree of Soil Saturation", + "modeling_realm": "land", + "ok_max_mean_abs": "1", + "ok_min_mean_abs": "0", + "out_name": "dos", + "standard_name": "", + "type": "real", + "units": "m3 m-3", + "valid_max": "2", + "valid_min": "0" + }, + "dosStderr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean area: mean where land", + "dimensions": "longitude latitude time", + "long_name": "Degree of Soil Saturation Error", + "modeling_realm": "land", + "out_name": "dosStderr", + "standard_name": "", + "type": "real", + "units": "m3 m-3", + "valid_max": "1.0", + "valid_min": "0.0" + }, + "dpn2o": { + "cell_measures": "area: areacella", + "cell_methods": "area: time: mean", + "comment": "Positive values correspond to higher partial pressure in sea water than air.", + "dimensions": "longitude latitude time", + "long_name": "Surface Nitrous Oxide (N2O) Partial Pressure Difference between Sea Water and Air", + "modeling_realm": "ocnBgchem", + "out_name": "dpn2o", + "standard_name": "", + "type": "real", + "units": "Pa" + }, + "et": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude time", + "long_name": "Evapotranspiration", + "modeling_realm": "atmos", + "out_name": "et", + "standard_name": "", + "type": "real", + "units": "mm day-1" + }, + "etStderr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "Standard deviation", + "dimensions": "longitude latitude time", + "long_name": "Evapotranspiration Error", + "modeling_realm": "land", + "out_name": "etStderr", + "standard_name": "", + "type": "real", + "units": "mm day-1", + "valid_min": "0" + }, + "fapar": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude time", + "long_name": "Fraction of Absorbed Photosynthetically Active Radiation", + "modeling_realm": "land", + "out_name": "fapar", + "standard_name": "", + "type": "real", + "units": "1" + }, + "gppStderr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "Standard deviation calculated based on median absolute deviation", + "dimensions": "longitude latitude time", + "long_name": "Carbon Mass Flux out of Atmosphere due to Gross Primary Production on Land Error", + "modeling_realm": "land", + "out_name": "gppStderr", + "standard_name": "", + "type": "real", + "units": "kg m-2 s-1", + "valid_min": "0" + }, + "hfns": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude time", + "long_name": "Surface Net Heat Flux", + "modeling_realm": "atmos", + "out_name": "hfns", + "positive": "up", + "standard_name": "", + "type": "real", + "units": "W m-2" + }, + "hurStderr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "The relative humidity with respect to liquid water for T> 0 C, and with respect to ice for T<0 C.", + "dimensions": "longitude latitude plev19 time", + "long_name": "Relative Humidity Error", + "modeling_realm": "atmos", + "out_name": "hurStderr", + "standard_name": "", + "type": "real", + "units": "%" + }, + "husStderr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude plev19 time", + "long_name": "Specific Humidity Error", + "modeling_realm": "atmos", + "ok_max_mean_abs": "0.01041", + "ok_min_mean_abs": "-0.0003539", + "out_name": "husStderr", + "standard_name": "", + "type": "real", + "units": "1", + "valid_max": "0.02841", + "valid_min": "-0.000299" + }, + "iwpStderr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude time", + "long_name": "Condensed Ice Path Error", + "modeling_realm": "atmos", + "ok_max_mean_abs": "1.0", + "ok_min_mean_abs": "0.0", + "out_name": "iwpStderr", + "standard_name": "", + "type": "real", + "units": "kg m-2", + "valid_max": "5.0", + "valid_min": "0.0" + }, + "lapserate": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "Atmospheric lapse rate calculated as -dT/dz in K per km.", + "dimensions": "longitude latitude plev19 time", + "long_name": "Lapse Rate", + "modeling_realm": "atmos", + "out_name": "lapserate", + "standard_name": "", + "type": "real", + "units": "K km-1" + }, + "lvp": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "longitude latitude time", + "long_name": "Latent Heat Release from Precipitation", + "modeling_realm": "atmos", + "out_name": "lvp", + "standard_name": "", + "type": "real", + "units": "W m-2" + }, + "lwcre": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at the top of the atmosphere (to be compared with satellite measurements)", + "dimensions": "longitude latitude time", + "long_name": "TOA Longwave Cloud Radiative Effect", + "modeling_realm": "atmos", + "out_name": "lwcre", + "positive": "down", + "standard_name": "", + "type": "real", + "units": "W m-2" + }, + "lweGrace": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean area: mean where land", + "comment": "liquid water equivalent thickness anomaly", + "dimensions": "longitude latitude time", + "long_name": "Liquid Water Equivalent Thickness Anomaly", + "modeling_realm": "land", + "out_name": "lweGrace", + "standard_name": "", + "type": "real", + "units": "m" + }, + "lwp": { + "cell_measures": "area: areacella", + "cell_methods": "area: time: mean", + "comment": "The total mass of liquid water in cloud per unit area.", + "dimensions": "longitude latitude time", + "long_name": "Liquid Water Path", + "modeling_realm": "aerosol", + "out_name": "lwp", + "standard_name": "atmosphere_mass_content_of_cloud_liquid_water", + "type": "real", + "units": "kg m-2" + }, + "lwpStderr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude time", + "long_name": "Liquid Water Path Error", + "modeling_realm": "atmos", + "ok_max_mean_abs": "1.0", + "ok_min_mean_abs": "0.0", + "out_name": "lwpStderr", + "standard_name": "", + "type": "real", + "units": "kg m-2", + "valid_max": "5.0", + "valid_min": "0.0" + }, + "n2oflux": { + "cell_measures": "area: areacella", + "cell_methods": "area: time: mean", + "comment": "Positive flux is into the atmosphere.", + "dimensions": "longitude latitude time", + "long_name": "Surface Upward Ocean to Atmosphere Mole Flux of Nitrous Oxide (N2O)", + "modeling_realm": "ocnBgchem", + "out_name": "n2oflux", + "positive": "up", + "standard_name": "", + "type": "real", + "units": "mol m-2 s-1" + }, + "n2os": { + "cell_measures": "area: areacella", + "cell_methods": "area: time: mean", + "comment": "As n2o, but only at the surface", + "dimensions": "longitude latitude time", + "long_name": "Mole Fraction of N2O at Surface", + "modeling_realm": "atmos", + "out_name": "n2os", + "standard_name": "mole_fraction_of_nitrous_oxide_in_air", + "type": "real", + "units": "mol mol-1" + }, + "netcre": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at the top of the atmosphere (to be compared with satellite measurements)", + "dimensions": "longitude latitude time", + "long_name": "TOA Net Cloud Radiative Effect", + "modeling_realm": "atmos", + "out_name": "netcre", + "positive": "down", + "standard_name": "", + "type": "real", + "units": "W m-2" + }, + "od550aerStderr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "AOD error from the ambient aerosls (i.e., includes aerosol water). Does not include AOD from stratospheric aerosols if these are prescribed but includes other possible background aerosol types.", + "dimensions": "longitude latitude time", + "long_name": "Ambient Aerosol Optical Thickness at 550 nm Error", + "modeling_realm": "aerosol", + "out_name": "od550aerStderr", + "standard_name": "", + "type": "real", + "units": "1" + }, + "od870aerStderr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "AOD error from the ambient aerosls (i.e., includes aerosol water). Does not include AOD from stratospheric aerosols if these are prescribed but includes other possible background aerosol types.", + "dimensions": "longitude latitude time", + "long_name": "Ambient Aerosol Optical Thickness at 870 nm Error", + "modeling_realm": "aerosol", + "out_name": "od870aerStderr", + "standard_name": "", + "type": "real", + "units": "1" + }, + "ohc": { + "cell_measures": "volume: volcello", + "cell_methods": "time: mean area: where sea", + "comment": "Heat content", + "dimensions": "longitude latitude time olevel", + "generic_levels": "olevel", + "long_name": "Heat content in grid cell", + "modeling_realm": "ocean", + "out_name": "ohc", + "standard_name": "", + "type": "real", + "units": "J" + }, + "prStderr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at surface; includes both liquid and solid phases from all types of clouds (both large-scale and convective)", + "dimensions": "longitude latitude time", + "long_name": "Precipitation Standard Error", + "modeling_realm": "atmos", + "ok_max_mean_abs": "0.001", + "ok_min_mean_abs": "0", + "out_name": "prStderr", + "standard_name": "", + "type": "real", + "units": "kg m-2 s-1", + "valid_max": "0.001", + "valid_min": "0" + }, + "prl": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "large scale precipitation at surface; includes both liquid and solid phases from all types of clouds (both large-scale and convective)", + "dimensions": "longitude latitude time", + "long_name": "Large Scale Precipitation", + "modeling_realm": "atmos", + "out_name": "prl", + "standard_name": "", + "type": "real", + "units": "kg m-2 s-1" + }, + "prodlnox": { + "comment": "Production NOX (NO+NO2) by lightning globally integrated", + "dimensions": "time", + "long_name": "Tendency of atmosphere mass content of NOx from lightning", + "modeling_realm": "atmos", + "out_name": "prodlnox", + "standard_name": "", + "type": "real", + "units": "kg s-1" + }, + "ptype": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "Description of numerical values can be found in GRIB2 - CODE TABLE 4.201", + "dimensions": "longitude latitude time", + "long_name": "Precipitation type", + "modeling_realm": "atmos", + "ok_max_mean_abs": "255", + "ok_min_mean_abs": "0", + "out_name": "ptype", + "standard_name": "", + "type": "real", + "units": "1", + "valid_max": "255", + "valid_min": "0" + }, + "qep": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "evspsbl-pr", + "dimensions": "longitude latitude time", + "long_name": "Net moisture flux into atmosphere", + "modeling_realm": "atmos", + "out_name": "qep", + "positive": "up", + "standard_name": "", + "type": "real", + "units": "kg m-2 s-1" + }, + "rlns": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "difference between downward and upward thermal radiation at the surface of the Earth", + "dimensions": "longitude latitude time", + "long_name": "Surface Net downward Longwave Radiation", + "modeling_realm": "atmos", + "out_name": "rlns", + "positive": "down", + "standard_name": "surface_net_downward_longwave_flux", + "type": "real", + "units": "W m-2" + }, + "rlnst": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "to surface and outer space", + "dimensions": "longitude latitude time", + "long_name": "Net Atmospheric Longwave Cooling", + "modeling_realm": "atmos", + "out_name": "rlnst", + "standard_name": "", + "type": "real", + "units": "W m-2" + }, + "rlnstcs": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "to surface and outer space (For definition see: DeAngelis et al. 2015)", + "dimensions": "longitude latitude time", + "long_name": "Net Atmospheric Longwave Cooling assuming clear sky", + "modeling_realm": "atmos", + "out_name": "rlnstcs", + "positive": "up", + "standard_name": "", + "type": "real", + "units": "W m-2" + }, + "rlntcs": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at the top of the atmosphere", + "dimensions": "longitude latitude time", + "long_name": "TOA Net downward Longwave Radiation assuming clear sky", + "modeling_realm": "atmos", + "out_name": "rlntcs", + "positive": "down", + "standard_name": "", + "type": "real", + "units": "W m-2" + }, + "rluscs": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude time", + "long_name": "Surface Upwelling Clear-Sky Longwave Radiation", + "modeling_realm": "atmos", + "ok_max_mean_abs": "376.3", + "ok_min_mean_abs": "325.6", + "out_name": "rluscs", + "positive": "up", + "standard_name": "", + "type": "real", + "units": "W m-2", + "valid_max": "658", + "valid_min": "43.75" + }, + "rlut": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at the top of the atmosphere (to be compared with satellite measurements)", + "dimensions": "longitude latitude time", + "long_name": "TOA Outgoing Longwave Radiation", + "modeling_realm": "atmos", + "ok_max_mean_abs": "234.4", + "ok_min_mean_abs": "207.4", + "out_name": "rlut", + "positive": "up", + "standard_name": "toa_outgoing_longwave_flux", + "type": "real", + "units": "W m-2", + "valid_max": "383.2", + "valid_min": "67.48" + }, + "rlutcs": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude time", + "long_name": "TOA Outgoing Clear-Sky Longwave Radiation", + "modeling_realm": "atmos", + "ok_max_mean_abs": "260.4", + "ok_min_mean_abs": "228.9", + "out_name": "rlutcs", + "positive": "up", + "standard_name": "toa_outgoing_longwave_flux_assuming_clear_sky", + "type": "real", + "units": "W m-2", + "valid_max": "377.5", + "valid_min": "70.59" + }, + "rsns": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "amount of solar radiation that reaches the surface of the Earth minus the amount reflected by the Earth's surface", + "dimensions": "longitude latitude time", + "long_name": "Surface Net downward Shortwave Radiation", + "modeling_realm": "atmos", + "out_name": "rsns", + "positive": "down", + "standard_name": "surface_net_downward_shortwave_flux", + "type": "real", + "units": "W m-2" + }, + "rsnst": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "(For definition see: DeAngelis et al. 2015)", + "dimensions": "longitude latitude time", + "long_name": "Heating from Shortwave Absorption", + "modeling_realm": "atmos", + "out_name": "rsnst", + "standard_name": "", + "type": "real", + "units": "W m-2" + }, + "rsnstcs": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "(For definition see: DeAngelis et al. 2015)", + "dimensions": "longitude latitude time", + "long_name": "Heating from Shortwave Absorption assuming clear sky", + "modeling_realm": "atmos", + "out_name": "rsnstcs", + "positive": "up", + "standard_name": "", + "type": "real", + "units": "W m-2" + }, + "rsnstcsnorm": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "(For definition see: DeAngelis et al. 2015)", + "dimensions": "longitude latitude time", + "long_name": "Heating from Shortwave Absorption assuming clear sky normalized by incoming solar radiation", + "modeling_realm": "atmos", + "out_name": "rsnstcsnorm", + "standard_name": "", + "type": "real", + "units": "%" + }, + "rsnt": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at the top of the atmosphere (to be compared with satellite measurements)", + "dimensions": "longitude latitude time", + "long_name": "TOA Net downward Shortwave Radiation", + "modeling_realm": "atmos", + "out_name": "rsnt", + "positive": "down", + "standard_name": "", + "type": "real", + "units": "W m-2" + }, + "rsntcs": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at the top of the atmosphere", + "dimensions": "longitude latitude time", + "long_name": "TOA Net downward Shortwave Radiation assuming clear sky", + "modeling_realm": "atmos", + "out_name": "rsntcs", + "positive": "down", + "standard_name": "", + "type": "real", + "units": "W m-2" + }, + "rsut": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at the top of the atmosphere", + "dimensions": "longitude latitude time", + "long_name": "TOA Outgoing Shortwave Radiation", + "modeling_realm": "atmos", + "ok_max_mean_abs": "114.1", + "ok_min_mean_abs": "96.72", + "out_name": "rsut", + "positive": "up", + "standard_name": "toa_outgoing_shortwave_flux", + "type": "real", + "units": "W m-2", + "valid_max": "421.9", + "valid_min": "-0.02689" + }, + "rsutcs": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude time", + "long_name": "TOA Outgoing Clear-Sky Shortwave Radiation", + "modeling_realm": "atmos", + "ok_max_mean_abs": "73.36", + "ok_min_mean_abs": "54.7", + "out_name": "rsutcs", + "positive": "up", + "standard_name": "toa_outgoing_shortwave_flux_assuming_clear_sky", + "type": "real", + "units": "W m-2", + "valid_max": "444", + "valid_min": "0" + }, + "rtnt": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at the top of the atmosphere (to be compared with satellite measurements)", + "dimensions": "longitude latitude time", + "long_name": "TOA Net downward Total Radiation", + "modeling_realm": "atmos", + "out_name": "rtnt", + "positive": "down", + "standard_name": "", + "type": "real", + "units": "W m-2" + }, + "rx1day": { + "cell_measures": "area: areacella", + "cell_methods": "time: sum within days time: maximum over days", + "comment": "ETCCDI (Extreme climate change index) annual/monthly maximum 1-day precipitation", + "dimensions": "longitude latitude time", + "frequency": "yr", + "long_name": "Annual/monthly maximum 1-day precipitation", + "modeling_realm": "ground", + "out_name": "rx1day", + "standard_name": "", + "type": "real", + "units": "mm" + }, + "rx5day": { + "cell_measures": "area: areacella", + "cell_methods": "time: sum within days time: maximum over days", + "comment": "ETCCDI (Extreme climate change index) annual/monthly maximum 5-day precipitation", + "dimensions": "longitude latitude time", + "frequency": "yr", + "long_name": "Annual/monthly maximum 5-day precipitation", + "modeling_realm": "ground", + "out_name": "rx5day", + "standard_name": "", + "type": "real", + "units": "mm" + }, + "siextent": { + "cell_measures": "area: areacello", + "cell_methods": "area: mean where sea time: mean", + "comment": "", + "dimensions": "longitude latitude time", + "long_name": "Sea Ice Extent", + "modeling_realm": "seaIce", + "out_name": "siextent", + "standard_name": "", + "type": "real", + "units": "m2", + "valid_max": "", + "valid_min": "" + }, + "sispeed": { + "cell_measures": "area: areacella", + "cell_methods": "area: time: mean where sea_ice (comment: mask=siconc)", + "comment": "Speed of ice (i.e. mean absolute velocity) to account for back-and-forth movement of the ice", + "dimensions": "longitude latitude time", + "long_name": "Sea-ice speed", + "modeling_realm": "seaIce", + "out_name": "sispeed", + "standard_name": "sea_ice_speed", + "type": "real", + "units": "m s-1", + "valid_max": "", + "valid_min": "" + }, + "sithick": { + "cell_measures": "area: areacella", + "cell_methods": "area: time: mean where sea_ice (comment: mask=siconc)", + "comment": "Actual (floe) thickness of sea ice (NOT volume divided by grid area as was done in CMIP5)", + "dimensions": "longitude latitude time", + "long_name": "Sea Ice Thickness", + "modeling_realm": "seaIce", + "out_name": "sithick", + "standard_name": "sea_ice_thickness", + "type": "real", + "units": "m", + "valid_max": "", + "valid_min": "" + }, + "sm": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean area: mean where land", + "comment": "the volume of water in all phases in a thin surface soil layer.", + "dimensions": "longitude latitude time", + "long_name": "Volumetric Moisture in Upper Portion of Soil Column", + "modeling_realm": "land", + "ok_max_mean_abs": "1", + "ok_min_mean_abs": "0", + "out_name": "sm", + "standard_name": "", + "type": "real", + "units": "m3 m-3", + "valid_max": "1", + "valid_min": "0" + }, + "sm1m": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean area: mean where land", + "comment": "the volume of water in all phases in the upper 1 metre of the soil column", + "dimensions": "longitude latitude time", + "long_name": "Volumetric Moisture in Upper 1 Metre of Soil Column", + "modeling_realm": "land", + "ok_max_mean_abs": "1", + "ok_min_mean_abs": "0", + "out_name": "sm", + "standard_name": "", + "type": "real", + "units": "m3 m-3", + "valid_max": "1", + "valid_min": "0" + }, + "smStderr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean area: mean where land", + "comment": "Error of the volume of water in all phases in a thin surface soil layer.", + "dimensions": "longitude latitude time", + "long_name": "Volumetric Moisture in Upper Portion of Soil Column Error", + "modeling_realm": "land", + "out_name": "smStderr", + "standard_name": "", + "type": "real", + "units": "m3 m-3", + "valid_max": "1.0", + "valid_min": "0.0" + }, + "soz": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "stratospheric ozone column calculated at 0 degrees C and 1 bar, such that 1m = 1e5 DU. Here, the stratosphere is defined as the region where O3 mole fraction >= 125 ppb.", + "dimensions": "longitude latitude time", + "long_name": "Stratospheric Ozone Column (O3 mole fraction >= 125 ppb)", + "modeling_realm": "atmos", + "out_name": "soz", + "standard_name": "equivalent_thickness_at_stp_of_atmosphere_ozone_content", + "type": "real", + "units": "m", + "valid_max": "5000.0", + "valid_min": "0.0" + }, + "swcre": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at the top of the atmosphere (to be compared with satellite measurements)", + "dimensions": "longitude latitude time", + "long_name": "TOA Shortwave Cloud Radiative Effect", + "modeling_realm": "atmos", + "out_name": "swcre", + "positive": "down", + "standard_name": "", + "type": "real", + "units": "W m-2" + }, + "tasConf5": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "time", + "long_name": "Near-Surface Air Temperature Uncertainty Range", + "modeling_realm": "atmos", + "ok_max_mean_abs": "20.", + "ok_min_mean_abs": "0", + "out_name": "tasConf5", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "20.", + "valid_min": "0" + }, + "tasConf95": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "time", + "long_name": "Near-Surface Air Temperature Uncertainty Range", + "modeling_realm": "atmos", + "ok_max_mean_abs": "20.", + "ok_min_mean_abs": "0", + "out_name": "tasConf95", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "20.", + "valid_min": "0" + }, + "tasa": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude time", + "long_name": "Near-Surface Air Temperature Anomaly", + "modeling_realm": "atmos", + "ok_max_mean_abs": "20.", + "ok_min_mean_abs": "-20", + "out_name": "tasa", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "20.0", + "valid_min": "-20.0" + }, + "tasaga": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "time", + "long_name": "Global-mean Near-Surface Air Temperature Anomaly", + "modeling_realm": "atmos", + "ok_max_mean_abs": "20.", + "ok_min_mean_abs": "-20", + "out_name": "tasaga", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "20.0", + "valid_min": "-20.0" + }, + "tcw": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude plev19 time", + "long_name": "Mass Fraction of Cloud Total Water (liquid + ice)", + "modeling_realm": "atmos", + "out_name": "tcw", + "standard_name": "", + "type": "real", + "units": "kg kg-1" + }, + "tnn": { + "cell_measures": "area: areacella", + "cell_methods": "time: minimum", + "comment": "ETCCDI (Extreme climate change index) annual/monthly minimum value of daily minimum temperature", + "dimensions": "longitude latitude time", + "frequency": "yr", + "long_name": "Annual/monthly minimum value of daily minimum temperature", + "modeling_realm": "ground", + "out_name": "tnn", + "standard_name": "", + "type": "real", + "units": "degrees_C" + }, + "tosStderr": { + "cell_measures": "area: areacello", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "longitude latitude time", + "long_name": "Sea Surface Temperature Error", + "modeling_realm": "ocean", + "ok_max_mean_abs": "", + "ok_min_mean_abs": "0", + "out_name": "tosStderr", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "", + "valid_min": "0" + }, + "toz": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "Total ozone column calculated at 0 degrees C and 1 bar, such that 1m = 1e5 DU.", + "dimensions": "longitude latitude time", + "long_name": "Total Ozone Column", + "modeling_realm": "atmos", + "out_name": "toz", + "standard_name": "equivalent_thickness_at_stp_of_atmosphere_ozone_content", + "type": "real", + "units": "m", + "valid_max": "5000.0", + "valid_min": "0.0" + }, + "tozStderr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "Total ozone column error calculated at 0 degrees C and 1 bar, such that 1m = 1e5 DU.", + "dimensions": "longitude latitude time", + "long_name": "Total Ozone Column Error", + "modeling_realm": "atmos", + "out_name": "tozStderr", + "standard_name": "equivalent_thickness_at_stp_of_atmosphere_ozone_content", + "type": "real", + "units": "m", + "valid_max": "5000.0", + "valid_min": "0.0" + }, + "tro3prof": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "latitude plev19 time", + "long_name": "Ozone Volume Mixing Ratio", + "modeling_realm": "atmos", + "out_name": "tro3prof", + "standard_name": "", + "type": "real", + "units": "1e-9", + "valid_max": "1.0", + "valid_min": "0.0" + }, + "tro3profStderr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "latitude plev19 time", + "long_name": "Ozone Volume Mixing Ratio Error", + "modeling_realm": "atmos", + "out_name": "tro3profStderr", + "standard_name": "", + "type": "real", + "units": "1e-9", + "valid_max": "1.0", + "valid_min": "0.0" + }, + "troz": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "tropospheric ozone column calculated at 0 degrees C and 1 bar, such that 1m = 1e5 DU. Here, the troposphere is defined as the region where O3 mole fraction < 125 ppb.", + "dimensions": "longitude latitude time", + "long_name": "Tropospheric Ozone Column (O3 mole fraction < 125 ppb)", + "modeling_realm": "atmos", + "out_name": "troz", + "standard_name": "equivalent_thickness_at_stp_of_atmosphere_ozone_content", + "type": "real", + "units": "m", + "valid_max": "5000.0", + "valid_min": "0.0" + }, + "tsDay": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time latitude longitude", + "long_name": "surface temperture daytime", + "modeling_realm": "atmos", + "out_name": "tsDay", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "350", + "valid_min": "190", + "var_name": "tsDay" + }, + "tsLCDay": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time latitude longitude", + "long_name": "land cover class daytime", + "modeling_realm": "atmos", + "out_name": "tsLCDay", + "standard_name": "", + "type": "float", + "units": "", + "valid_max": "255", + "valid_min": "0", + "var_name": "tsLCDay" + }, + "tsLCNight": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time latitude longitude", + "long_name": "land cover class night time", + "modeling_realm": "atmos", + "out_name": "tsLCNight", + "standard_name": "", + "type": "float", + "units": "", + "valid_max": "255", + "valid_min": "0", + "var_name": "tsLCNight" + }, + "tsLSSysErrDay": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time", + "long_name": "uncertainty from large-scale systematic errors daytime", + "modeling_realm": "atmos", + "out_name": "tsLSSysErrDay", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "10", + "valid_min": "0", + "var_name": "tsLSSysErrDay" + }, + "tsLSSysErrNight": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time", + "long_name": "uncertainty from large-scale systematic errors night time", + "modeling_realm": "atmos", + "out_name": "tsLSSysErrNight", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "10", + "valid_min": "0", + "var_name": "tsLSSysErrNight" + }, + "tsLocalAtmErrDay": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time latitude longitude", + "long_name": "uncertainty from locally correlated errors on atmospheric scales daytime", + "modeling_realm": "atmos", + "out_name": "tsLocalAtmErrDay", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "10", + "valid_min": "0", + "var_name": "tsLocalAtmErrDay" + }, + "tsLocalAtmErrNight": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time latitude longitude", + "long_name": "uncertainty from locally correlated errors on atmospheric scales night time", + "modeling_realm": "atmos", + "out_name": "tsLocalAtmErrNight", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "10", + "valid_min": "0", + "var_name": "tsLocalAtmErrNight" + }, + "tsLocalSfcErrDay": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time latitude longitude", + "long_name": "uncertainty from locally correlated errors on surface scales daytime", + "modeling_realm": "atmos", + "out_name": "tsLocalSfcErrDay", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "10", + "valid_min": "0", + "var_name": "tsLocalSfcErrDay" + }, + "tsLocalSfcErrNight": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time latitude longitude", + "long_name": "uncertainty from locally correlated errors on surface scales night time", + "modeling_realm": "atmos", + "out_name": "tsLocalSfcErrNight", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "10", + "valid_min": "0", + "var_name": "tsLocalSfcErrNight" + }, + "tsNight": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time latitude longitude", + "long_name": "surface temperature nighttime", + "modeling_realm": "atmos", + "out_name": "tsNight", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "350", + "valid_min": "190", + "var_name": "tsNight" + }, + "tsStderr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "\"\"skin\"\" temperature error (i.e., SST for open ocean)", + "dimensions": "longitude latitude time", + "long_name": "Surface Temperature Error", + "modeling_realm": "atmos", + "ok_max_mean_abs": "10", + "ok_min_mean_abs": "0", + "out_name": "tsStderr", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "10", + "valid_min": "0" + }, + "tsTotalDay": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time latitude longitude", + "long_name": "total uncertainty of land surface temperature daytime", + "modeling_realm": "atmos", + "out_name": "tsTotalDay", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "10", + "valid_min": "0", + "var_name": "tsTotalDay" + }, + "tsTotalNight": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time latitude longitude", + "long_name": "total uncertainty of land surface temperature night time", + "modeling_realm": "atmos", + "out_name": "tsTotalNight", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "10", + "valid_min": "0", + "var_name": "tsTotalNight" + }, + "tsUnCorErrDay": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time latitude longitude", + "long_name": "uncertainty from uncorrelated errors daytime", + "modeling_realm": "atmos", + "out_name": "tsUnCorErrDay", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "10", + "valid_min": "0", + "var_name": "tsUnCorErrDay" + }, + "tsUnCorErrNight": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time latitude longitude", + "long_name": "uncertainty from uncorrelated errors night time", + "modeling_realm": "atmos", + "out_name": "tsUnCorErrNight", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "10", + "valid_min": "0", + "var_name": "tsUnCorErrNight" + }, + "tsVarDay": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time latitude longitude", + "long_name": "land surface temperature variance daytime", + "modeling_realm": "atmos", + "out_name": "tsVarDay", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "100", + "valid_min": "0", + "var_name": "tsVarDay" + }, + "tsVarNight": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time latitude longitude", + "long_name": "land surface temperature variance night time", + "modeling_realm": "atmos", + "out_name": "tsVarNight", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "100", + "valid_min": "0", + "var_name": "tsVarNight" + }, + "txx": { + "cell_measures": "area: areacella", + "cell_methods": "time: maximum", + "comment": "ETCCDI (Extreme climate change index) annual/monthly maximum value of daily maximum temperature", + "dimensions": "longitude latitude time", + "frequency": "yr", + "long_name": "Annual/monthly maximum value of daily maximum temperature", + "modeling_realm": "ground", + "out_name": "txx", + "standard_name": "", + "type": "real", + "units": "degrees_C" + }, + "uajet": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "time", + "long_name": "Jet position expressed as latitude of maximum meridional wind speed", + "modeling_realm": "atmos", + "out_name": "uajet", + "standard_name": "", + "type": "real", + "units": "degrees" + }, + "vegfrac": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean area: where land", + "comment": "Fraction of entire grid cell that is not covered by bare soil.", + "dimensions": "longitude latitude time", + "long_name": "Vegetation Fraction", + "modeling_realm": "land", + "out_name": "vegfrac", + "standard_name": "", + "type": "real", + "units": "%" + }, + "xch4": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "Satellite retrieved column-average dry-air mole fraction of atmospheric methane (XCH4)", + "dimensions": "longitude latitude time", + "long_name": "Column-average Dry-air Mole Fraction of Atmospheric Methane", + "modeling_realm": "atmos", + "out_name": "xch4", + "standard_name": "", + "type": "real", + "units": "1" + }, + "xco2": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "Satellite retrieved column-average dry-air mole fraction of atmospheric carbon dioxide (XCO2)", + "dimensions": "longitude latitude time", + "long_name": "Column-average Dry-air Mole Fraction of Atmospheric Carbon Dioxide", + "modeling_realm": "atmos", + "out_name": "xco2", + "standard_name": "", + "type": "real", + "units": "1" + } + } +} diff --git a/esmvalcore/cmor/tables/cmip6-custom/convert-cmip5-to-cmip6.py b/esmvalcore/cmor/tables/cmip6-custom/convert-cmip5-to-cmip6.py new file mode 100644 index 0000000000..4ef96ebaa4 --- /dev/null +++ b/esmvalcore/cmor/tables/cmip6-custom/convert-cmip5-to-cmip6.py @@ -0,0 +1,60 @@ +"""Convert CMIP5-style custom CMOR tables to a CMIP6-style custom table. + +Example usage: `python convert-cmip5-to-cmip6.py esmvalcore/cmor/tables/cmip5-custom/*.dat` +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from collections.abc import Iterable + + +def read(file: Path) -> dict[str, str]: + """Read a CMIP5-style custom CMOR table file into a `dict`.""" + result = {} + for line in file.read_text(encoding="utf-8").split("\n"): + if not line or line.startswith("!"): + continue + key, value = [elem.strip() for elem in line.split(":", 1)] + result[key] = value + return result + + +def translate(files: Iterable[Path]) -> dict[str, Any]: + """Read in CMIP5-style custom CMOR table files and return a CMIP6-style custom table.""" + result = { + "Header": { + "table_id": "Table custom", + "generic_levels": "olevel", + }, + "variable_entry": {}, + } + for file in files: + # Skip the coordinates file and use standard CMIP6 coordinates instead. + if "coordinates" in file.name: + continue + variable = read(file) + variable_entry = variable.pop("variable_entry") + # Remove the "SOURCE" key which has no meaning in CMOR tables. + variable.pop("SOURCE", None) + # Some files are missing `out_name`, assume it is the same as the entry. + if "out_name" not in variable: + variable["out_name"] = variable_entry + # Use a CMIP6 pressure levels coordinate. + if "plevs" in variable["dimensions"]: + variable["dimensions"] = variable["dimensions"].replace( + "plevs", + "plev19", + ) + result["variable_entry"][variable_entry] = variable + return result + + +if __name__ == "__main__": + table_files = [Path(p) for p in sys.argv[1:]] + print(json.dumps(translate(table_files), indent=4, sort_keys=True)) # noqa: T201 diff --git a/esmvalcore/cmor/tables/cmip7/Tables/CMIP7_aerosol.json b/esmvalcore/cmor/tables/cmip7/tables/CMIP7_aerosol.json similarity index 100% rename from esmvalcore/cmor/tables/cmip7/Tables/CMIP7_aerosol.json rename to esmvalcore/cmor/tables/cmip7/tables/CMIP7_aerosol.json diff --git a/esmvalcore/cmor/tables/cmip7/Tables/CMIP7_atmos.json b/esmvalcore/cmor/tables/cmip7/tables/CMIP7_atmos.json similarity index 100% rename from esmvalcore/cmor/tables/cmip7/Tables/CMIP7_atmos.json rename to esmvalcore/cmor/tables/cmip7/tables/CMIP7_atmos.json diff --git a/esmvalcore/cmor/tables/cmip7/Tables/CMIP7_atmosChem.json b/esmvalcore/cmor/tables/cmip7/tables/CMIP7_atmosChem.json similarity index 100% rename from esmvalcore/cmor/tables/cmip7/Tables/CMIP7_atmosChem.json rename to esmvalcore/cmor/tables/cmip7/tables/CMIP7_atmosChem.json diff --git a/esmvalcore/cmor/tables/cmip7/Tables/CMIP7_cell_measures.json b/esmvalcore/cmor/tables/cmip7/tables/CMIP7_cell_measures.json similarity index 100% rename from esmvalcore/cmor/tables/cmip7/Tables/CMIP7_cell_measures.json rename to esmvalcore/cmor/tables/cmip7/tables/CMIP7_cell_measures.json diff --git a/esmvalcore/cmor/tables/cmip7/Tables/CMIP7_coordinate.json b/esmvalcore/cmor/tables/cmip7/tables/CMIP7_coordinate.json similarity index 100% rename from esmvalcore/cmor/tables/cmip7/Tables/CMIP7_coordinate.json rename to esmvalcore/cmor/tables/cmip7/tables/CMIP7_coordinate.json diff --git a/esmvalcore/cmor/tables/cmip7/Tables/CMIP7_formula_terms.json b/esmvalcore/cmor/tables/cmip7/tables/CMIP7_formula_terms.json similarity index 100% rename from esmvalcore/cmor/tables/cmip7/Tables/CMIP7_formula_terms.json rename to esmvalcore/cmor/tables/cmip7/tables/CMIP7_formula_terms.json diff --git a/esmvalcore/cmor/tables/cmip7/Tables/CMIP7_grids.json b/esmvalcore/cmor/tables/cmip7/tables/CMIP7_grids.json similarity index 100% rename from esmvalcore/cmor/tables/cmip7/Tables/CMIP7_grids.json rename to esmvalcore/cmor/tables/cmip7/tables/CMIP7_grids.json diff --git a/esmvalcore/cmor/tables/cmip7/Tables/CMIP7_land.json b/esmvalcore/cmor/tables/cmip7/tables/CMIP7_land.json similarity index 100% rename from esmvalcore/cmor/tables/cmip7/Tables/CMIP7_land.json rename to esmvalcore/cmor/tables/cmip7/tables/CMIP7_land.json diff --git a/esmvalcore/cmor/tables/cmip7/Tables/CMIP7_landIce.json b/esmvalcore/cmor/tables/cmip7/tables/CMIP7_landIce.json similarity index 100% rename from esmvalcore/cmor/tables/cmip7/Tables/CMIP7_landIce.json rename to esmvalcore/cmor/tables/cmip7/tables/CMIP7_landIce.json diff --git a/esmvalcore/cmor/tables/cmip7/Tables/CMIP7_long_name_overrides.json b/esmvalcore/cmor/tables/cmip7/tables/CMIP7_long_name_overrides.json similarity index 100% rename from esmvalcore/cmor/tables/cmip7/Tables/CMIP7_long_name_overrides.json rename to esmvalcore/cmor/tables/cmip7/tables/CMIP7_long_name_overrides.json diff --git a/esmvalcore/cmor/tables/cmip7/Tables/CMIP7_ocean.json b/esmvalcore/cmor/tables/cmip7/tables/CMIP7_ocean.json similarity index 100% rename from esmvalcore/cmor/tables/cmip7/Tables/CMIP7_ocean.json rename to esmvalcore/cmor/tables/cmip7/tables/CMIP7_ocean.json diff --git a/esmvalcore/cmor/tables/cmip7/Tables/CMIP7_ocnBgchem.json b/esmvalcore/cmor/tables/cmip7/tables/CMIP7_ocnBgchem.json similarity index 100% rename from esmvalcore/cmor/tables/cmip7/Tables/CMIP7_ocnBgchem.json rename to esmvalcore/cmor/tables/cmip7/tables/CMIP7_ocnBgchem.json diff --git a/esmvalcore/cmor/tables/cmip7/Tables/CMIP7_seaIce.json b/esmvalcore/cmor/tables/cmip7/tables/CMIP7_seaIce.json similarity index 100% rename from esmvalcore/cmor/tables/cmip7/Tables/CMIP7_seaIce.json rename to esmvalcore/cmor/tables/cmip7/tables/CMIP7_seaIce.json diff --git a/esmvalcore/config/__init__.py b/esmvalcore/config/__init__.py index 5d23c6b0e2..333c3e30e1 100644 --- a/esmvalcore/config/__init__.py +++ b/esmvalcore/config/__init__.py @@ -13,10 +13,22 @@ """ -from ._config_object import CFG, Config, Session +import contextlib + +import iris + +from esmvalcore.config._config_object import CFG, Config, Session __all__ = ( "CFG", "Config", "Session", ) + +# Set iris.FUTURE flags +for attr, value in { + "save_split_attrs": True, + "date_microseconds": True, +}.items(): + with contextlib.suppress(AttributeError): + setattr(iris.FUTURE, attr, value) diff --git a/esmvalcore/config/_config.py b/esmvalcore/config/_config.py index 3003c6c0e6..9d9ff9d6b5 100644 --- a/esmvalcore/config/_config.py +++ b/esmvalcore/config/_config.py @@ -3,7 +3,6 @@ from __future__ import annotations import collections.abc -import contextlib import logging import os import warnings @@ -11,10 +10,9 @@ from pathlib import Path from typing import TYPE_CHECKING, Any -import iris import yaml -from esmvalcore.cmor.table import CMOR_TABLES, read_cmor_tables +from esmvalcore.cmor.table import read_cmor_tables from esmvalcore.exceptions import ESMValCoreDeprecationWarning, RecipeError if TYPE_CHECKING: @@ -30,15 +28,6 @@ USER_EXTRA_FACETS = Path.home() / ".esmvaltool" / "extra_facets" -# Set iris.FUTURE flags -for attr, value in { - "save_split_attrs": True, - "date_microseconds": True, -}.items(): - with contextlib.suppress(AttributeError): - setattr(iris.FUTURE, attr, value) - - # TODO: remove in v2.15.0 def _deep_update(dictionary, update): for key, value in update.items(): @@ -134,28 +123,6 @@ def get_project_config(project): raise RecipeError(msg) -def get_institutes(variable): - """Return the institutes given the dataset name in CMIP6.""" - dataset = variable["dataset"] - project = variable["project"] - try: - return CMOR_TABLES[project].institutes[dataset] - except (KeyError, AttributeError): - return [] - - -def get_activity(variable): - """Return the activity given the experiment name in CMIP6.""" - project = variable["project"] - try: - exp = variable["exp"] - if isinstance(exp, list): - return [CMOR_TABLES[project].activities[value][0] for value in exp] - return CMOR_TABLES[project].activities[exp][0] - except (KeyError, AttributeError): - return None - - def get_ignored_warnings(project: FacetValue, step: str) -> None | list: """Get ignored warnings for a given preprocessing step.""" if project not in CFG: diff --git a/esmvalcore/config/_config_validators.py b/esmvalcore/config/_config_validators.py index 272f6f56e0..33a94ff36f 100644 --- a/esmvalcore/config/_config_validators.py +++ b/esmvalcore/config/_config_validators.py @@ -7,7 +7,6 @@ import warnings from collections.abc import Iterable from functools import lru_cache, partial -from importlib.resources import files as importlib_files from pathlib import Path from typing import TYPE_CHECKING, Any, Literal @@ -15,6 +14,7 @@ from esmvalcore import __version__ as current_version from esmvalcore.cmor.check import CheckLevels +from esmvalcore.cmor.table import load_cmor_tables from esmvalcore.config._config import TASKSEP, load_config_developer from esmvalcore.exceptions import ( ESMValCoreDeprecationWarning, @@ -285,9 +285,10 @@ def validate_drs(value): def validate_config_developer(value): """Validate and load config developer path.""" path = validate_path_or_none(value) - if path is None: - path = importlib_files("esmvalcore") / "config-developer.yml" - load_config_developer(path) + if path is not None: + # This has the side-effect of updating `esmvalcore.config._config.CFG` + # `esmvalcore.cmor.tables.CMOR_TABLES`. + load_config_developer(path) return path @@ -373,12 +374,29 @@ def validate_extra_facets_dir(value): return validate_pathlist(value) +def validate_cmor_tables(value: dict) -> None: + """Validate the CMOR table configuration.""" + # This has the side-effect of updating `esmvalcore.cmor.tables.CMOR_TABLES`. + # + # Relying on global state is not nice, preferably we should get rid of any + # global state except the defaults for starting a new session in + # `esmvalcore.config.CFG`. + # + # Having side effects when updating an `esmvalcore.config.Session` object + # that changes global state of the `esmvalcore` package is nasty and should + # preferably be avoided. This would require passing around session objects + # instead of relying on global state (e.g. esmvalcore.config.CFG, + # esmvalcore.cmor.tables.CMOR_TABLES). + load_cmor_tables({"projects": value}) # type: ignore[arg-type] + + def validate_projects( value: dict, ) -> dict[str, dict[str, Any]]: """Validate projects mapping.""" mapping = validate_dict(value) options_for_project: dict[str, Callable[[Any], Any]] = { + "cmor_table": validate_dict, "data": validate_dict, # TODO: try to create data sources here "extra_facets": validate_dict, "preprocessor_filename_template": validate_string, @@ -393,6 +411,7 @@ def validate_projects( ) raise ValidationError(msg) from None mapping[project][option] = options_for_project[option](val) + validate_cmor_tables(mapping) return mapping diff --git a/esmvalcore/config/configurations/defaults/cmor_tables.yml b/esmvalcore/config/configurations/defaults/cmor_tables.yml new file mode 100644 index 0000000000..06ec45c667 --- /dev/null +++ b/esmvalcore/config/configurations/defaults/cmor_tables.yml @@ -0,0 +1,77 @@ +# CMOR table configuration. +projects: + # Projects hosted on ESGF. + CMIP7: + cmor_table: + type: esmvalcore.cmor.table.CMIP6Info + paths: + - cmip7/tables + - cmip6-custom + CMIP6: + cmor_table: &cmip6 + type: esmvalcore.cmor.table.CMIP6Info + paths: + - cmip6/Tables + - cmip6-custom + CMIP5: + cmor_table: &cmip5 + type: esmvalcore.cmor.table.CMIP5Info + paths: + - cmip5/Tables + - cmip5-custom + CMIP3: + cmor_table: + type: esmvalcore.cmor.table.CMIP3Info + paths: + - cmip3/Tables + - cmip5-custom + CORDEX: + cmor_table: + type: esmvalcore.cmor.table.CMIP5Info + paths: + - cordex/Tables + - cmip5-custom + obs4MIPs: + cmor_table: + type: esmvalcore.cmor.table.Obs4MIPsInfo + # Set `strict: false` because the tables are incomplete. + strict: false + paths: + - obs4mips/Tables + - cmip6-custom + ana4MIPs: + # CMOR tables are available at https://github.com/PCMDI/ana4MIPs-cmor-tables/ + # but it is not clear how well these match the data. + cmor_table: + <<: *cmip5 + strict: false + # Observational and reanalysis data that can be read in its native format by ESMValCore. + native6: + cmor_table: &native6 + <<: *cmip6 + strict: false + # Data from selected climate models that can be read in its native format by ESMValCore. + ACCESS: + cmor_table: + <<: *native6 + CESM: + cmor_table: + <<: *native6 + EMAC: + cmor_table: + <<: *native6 + ICON: + cmor_table: + <<: *native6 + IPSLCM: + cmor_table: + <<: *native6 + # Data that has been CMORized by ESMValTool + OBS6: + cmor_table: + <<: *cmip6 + strict: false + OBS: + cmor_table: + <<: *cmip5 + strict: false diff --git a/esmvalcore/dataset.py b/esmvalcore/dataset.py index 41690dd8a9..8a6a20c156 100644 --- a/esmvalcore/dataset.py +++ b/esmvalcore/dataset.py @@ -24,11 +24,7 @@ _update_cmor_facets, ) from esmvalcore.config import CFG -from esmvalcore.config._config import ( - get_activity, - get_institutes, - load_extra_facets, -) +from esmvalcore.config._config import load_extra_facets from esmvalcore.config._data_sources import _get_data_sources from esmvalcore.exceptions import InputFilesNotFound, RecipeError from esmvalcore.io.local import _dates_to_timerange @@ -756,14 +752,6 @@ def _get_extra_facets(self) -> dict[str, Any]: def _augment_facets(self) -> None: extra_facets = self._get_extra_facets() _augment(self.facets, extra_facets) - if "institute" not in self.facets: - institute = get_institutes(self.facets) - if institute: - self.facets["institute"] = institute - if "activity" not in self.facets: - activity = get_activity(self.facets) - if activity: - self.facets["activity"] = activity _update_cmor_facets(self.facets) if self.facets.get("frequency") == "fx": self.facets.pop("timerange", None) diff --git a/esmvalcore/io/__init__.py b/esmvalcore/io/__init__.py index e115c462f2..5a936d47e0 100644 --- a/esmvalcore/io/__init__.py +++ b/esmvalcore/io/__init__.py @@ -67,8 +67,8 @@ def load_data_sources( If no ``priority`` is configured for a data source, the default priority of 1 is used. - Arguments - --------- + Parameters + ---------- session: The configuration. project: diff --git a/esmvalcore/io/local.py b/esmvalcore/io/local.py index 56d0da90da..aad0d7dccf 100644 --- a/esmvalcore/io/local.py +++ b/esmvalcore/io/local.py @@ -600,6 +600,9 @@ def find_data(self, **facets: FacetValue) -> list[LocalFile]: f" within the requested timerange {facets['timerange']}" ) + if files: + self.debug_info = f"F{self.debug_info[len('No f') :]}" + return files def _path2facets(self, path: Path, add_timerange: bool) -> dict[str, str]: diff --git a/tests/integration/cmor/_fixes/native6/test_era5.py b/tests/integration/cmor/_fixes/native6/test_era5.py index b183bd811b..cb7333fd7c 100644 --- a/tests/integration/cmor/_fixes/native6/test_era5.py +++ b/tests/integration/cmor/_fixes/native6/test_era5.py @@ -674,8 +674,6 @@ def ptype_cmor_e1hr(): ], attributes={"comment": COMMENT}, ) - cube.coord("latitude").long_name = "latitude" - cube.coord("longitude").long_name = "longitude" return CubeList([cube]) @@ -752,8 +750,6 @@ def rlns_cmor_e1hr(): ], attributes={"comment": COMMENT, "positive": "down"}, ) - cube.coord("latitude").long_name = "latitude" # from custom table - cube.coord("longitude").long_name = "longitude" # from custom table return CubeList([cube]) @@ -984,8 +980,6 @@ def rsns_cmor_e1hr(): ], attributes={"comment": COMMENT, "positive": "down"}, ) - cube.coord("latitude").long_name = "latitude" # from custom table - cube.coord("longitude").long_name = "longitude" # from custom table return CubeList([cube]) diff --git a/tests/integration/cmor/test_read_cmor_tables.py b/tests/integration/cmor/test_read_cmor_tables.py index 86c90454f4..efd296219b 100644 --- a/tests/integration/cmor/test_read_cmor_tables.py +++ b/tests/integration/cmor/test_read_cmor_tables.py @@ -1,12 +1,23 @@ +from __future__ import annotations + from pathlib import Path from textwrap import dedent +from typing import TYPE_CHECKING import pytest import yaml -from esmvalcore.cmor.table import CMOR_TABLES, read_cmor_tables +from esmvalcore.cmor.table import ( + CMOR_TABLES, + VariableInfo, + get_tables, + read_cmor_tables, +) from esmvalcore.cmor.table import __file__ as root +if TYPE_CHECKING: + from esmvalcore.config import Session + def test_read_cmor_tables_raiser(): """Test func raiser.""" @@ -22,32 +33,94 @@ def test_read_cmor_tables(): for project in "CMIP5", "CMIP6": table = CMOR_TABLES[project] - assert ( - Path(table._cmor_folder) == table_path / project.lower() / "Tables" + assert table.paths == ( + table_path / project.lower() / "Tables", + table_path / f"{project.lower()}-custom", ) assert table.strict is True project = "OBS" table = CMOR_TABLES[project] - assert Path(table._cmor_folder) == table_path / "cmip5" / "Tables" + assert table.paths == ( + table_path / "cmip5" / "Tables", + table_path / "cmip5-custom", + ) assert table.strict is False project = "OBS6" table = CMOR_TABLES[project] - assert Path(table._cmor_folder) == table_path / "cmip6" / "Tables" + assert table.paths == ( + table_path / "cmip6" / "Tables", + table_path / "cmip6-custom", + ) assert table.strict is False project = "obs4MIPs" table = CMOR_TABLES[project] - assert Path(table._cmor_folder) == table_path / "obs4mips" / "Tables" + assert table.paths == ( + table_path / "obs4mips" / "Tables", + table_path / "cmip6-custom", + ) assert table.strict is False - project = "custom" - table = CMOR_TABLES[project] - assert Path(table._cmor_folder) == table_path / "custom" - assert table._user_table_folder is None - assert table.coords - assert table.tables["custom"] + +@pytest.mark.parametrize( + ( + "project", + "mip", + "short_name", + "branding_suffix", + ), + [ + ("CMIP7", "atmos", "tas", "tavg-h2m-hxy-u"), + ("CMIP7", "Amon", "alb", None), # custom derived variable + ("CMIP6", "Amon", "tas", None), + ("CMIP6", "Amon", "alb", None), # custom derived variable + ("CMIP6", "Amon", "ch4", "Clim"), # table entry != short_name + ("CMIP5", "Amon", "tas", None), + ("CMIP5", "Amon", "alb", None), # custom derived variable + ("CMIP3", "A1", "tas", None), + ("CMIP3", "A1", "alb", None), # custom derived variable + ("CORDEX", "mon", "tas", None), + ("CORDEX", "mon", "alb", None), # custom derived variable + ("obs4MIPs", "Amon", "tas", None), + ("obs4MIPs", "Amon", "agb", None), # custom variable + ("obs4MIPs", "Amon", "alb", None), # custom derived variable + ("ana4MIPs", "Amon", "tas", None), + ("native6", "Amon", "tas", None), + ("native6", "Amon", "agb", None), # custom variable + ("native6", "Amon", "alb", None), # custom derived variable + ("ACCESS", "Amon", "tas", None), + ("CESM", "Amon", "tas", None), + ("EMAC", "Amon", "tas", None), + ("ICON", "Amon", "tas", None), + ("IPSLCM", "Amon", "tas", None), + ("OBS6", "Amon", "tas", None), + ("OBS6", "Amon", "agb", None), # custom variable + ("OBS6", "Amon", "alb", None), # custom derived variable + ("OBS", "Amon", "tas", None), + ("OBS", "Amon", "agb", None), # custom variable + ("OBS", "Amon", "alb", None), # custom derived variable + ], +) +def test_get_tables( + session: Session, + project: str, + mip: str, + short_name: str, + branding_suffix: str | None, +) -> None: + info = get_tables(session, project) + assert info.tables + vardef = info.get_variable( + mip, + short_name, + branding_suffix=branding_suffix, + derived=short_name == "alb", + ) + assert isinstance(vardef, VariableInfo) + assert vardef.short_name + assert vardef.units CMOR_NEWVAR_ENTRY = dedent( @@ -125,7 +198,7 @@ def test_read_cmor_tables(): ) -def test_read_custom_cmor_tables(tmp_path): +def test_read_custom_cmor_tables_config_developer(tmp_path): """Test reading of custom CMOR tables.""" (tmp_path / "CMOR_newvarfortesting.dat").write_text(CMOR_NEWVAR_ENTRY) (tmp_path / "CMOR_netcre.dat").write_text(CMOR_NETCRE_ENTRY) @@ -152,10 +225,10 @@ def test_read_custom_cmor_tables(tmp_path): assert "custom" in CMOR_TABLES custom_table = CMOR_TABLES["custom"] - assert custom_table._cmor_folder == str( - Path(root).parent / "tables" / "custom", + assert custom_table.paths == ( + Path(root).parent / "tables" / "cmip5-custom", + tmp_path, ) - assert custom_table._user_table_folder == str(tmp_path) # Make sure that default tables have been read assert "alb" in custom_table.tables["custom"] diff --git a/tests/integration/cmor/test_table.py b/tests/integration/cmor/test_table.py index 08f4e76795..44feca13c5 100644 --- a/tests/integration/cmor/test_table.py +++ b/tests/integration/cmor/test_table.py @@ -12,6 +12,7 @@ CMIP5Info, CMIP6Info, CustomInfo, + Obs4MIPsInfo, _get_branding_suffixes, _get_mips, _update_cmor_facets, @@ -24,6 +25,8 @@ def test_update_cmor_facets(): "project": "CMIP6", "mip": "Amon", "short_name": "tas", + "dataset": "CanESM5", + "exp": "historical", } _update_cmor_facets(facets) @@ -32,12 +35,18 @@ def test_update_cmor_facets(): "project": "CMIP6", "mip": "Amon", "short_name": "tas", + "dataset": "CanESM5", "original_short_name": "tas", "standard_name": "air_temperature", "long_name": "Near-Surface Air Temperature", "units": "K", "modeling_realm": ["atmos"], "frequency": "mon", + "activity": "CMIP", + "exp": "historical", + "institute": [ + "CCCma", + ], } assert facets == expected @@ -76,12 +85,9 @@ def setUpClass(cls): We read CMIP6Info once to keep tests times manageable """ cls.variables_info = CMIP6Info( - "cmip6", - default=CustomInfo(), - strict=True, - alt_names=[ - ["sic", "siconc"], - ["tro3", "o3"], + paths=[ + Path("cmip6/Tables"), + Path("cmip6-custom"), ], ) @@ -177,11 +183,12 @@ def setUpClass(cls): We read CMIP6Info once to keep tests times manageable """ - cls.variables_info = CMIP6Info( - cmor_tables_path="obs4mips", - default=CustomInfo(), + cls.variables_info = Obs4MIPsInfo( + paths=[ + Path("obs4mips/Tables"), + Path("cmip6-custom"), + ], strict=False, - default_table_prefix="obs4MIPs_", ) def setUp(self): @@ -190,7 +197,7 @@ def setUp(self): def test_get_table_frequency(self): """Test get table frequency.""" self.assertEqual( - self.variables_info.get_table("obs4MIPs_monStderr").frequency, + self.variables_info.get_table("monStderr").frequency, "mon", ) @@ -215,7 +222,7 @@ def test_get_variable_ndvistderr(self): def test_get_variable_hus(self): """Get hus variable.""" - var = self.variables_info.get_variable("obs4MIPs_Amon", "hus") + var = self.variables_info.get_variable("Amon", "hus") self.assertEqual(var.short_name, "hus") self.assertEqual(var.frequency, "mon") @@ -231,7 +238,7 @@ def test_get_variable_from_custom(self): Note table name obs4MIPs_[mip] """ var = self.variables_info.get_variable( - "obs4MIPs_monStderr", + "monStderr", "prStderr", ) self.assertEqual(var.short_name, "prStderr") @@ -240,7 +247,7 @@ def test_get_variable_from_custom(self): def test_get_variable_from_custom_deriving(self): """Get a variable from default.""" var = self.variables_info.get_variable( - "obs4MIPs_Amon", + "Amon", "swcre", derived=True, ) @@ -248,7 +255,7 @@ def test_get_variable_from_custom_deriving(self): self.assertEqual(var.frequency, "mon") var = self.variables_info.get_variable( - "obs4MIPs_Aday", + "Aday", "swcre", derived=True, ) @@ -269,7 +276,13 @@ def setUpClass(cls): We read CMIP5Info once to keep testing times manageable """ - cls.variables_info = CMIP5Info("cmip5", CustomInfo(), strict=True) + cls.variables_info = CMIP5Info( + paths=[ + Path("cmip5/Tables"), + Path("cmip5-custom"), + ], + strict=True, + ) def setUp(self): self.variables_info.strict = True @@ -356,7 +369,13 @@ def setUpClass(cls): We read CMIP5Info once to keep testing times manageable """ - cls.variables_info = CMIP3Info("cmip3", CustomInfo(), strict=True) + cls.variables_info = CMIP3Info( + paths=[ + Path("cmip3/Tables"), + Path("cmip5-custom"), + ], + strict=True, + ) def setUp(self): self.variables_info.strict = True @@ -443,7 +462,12 @@ def setUpClass(cls): We read CORDEX once to keep testing times manageable """ - cls.variables_info = CMIP5Info("cordex", default=CustomInfo()) + cls.variables_info = CMIP5Info( + paths=[ + Path("cordex/Tables"), + Path("cmip5-custom"), + ], + ) def test_custom_tables_location(self): """Test constructor with custom tables location.""" @@ -478,23 +502,29 @@ def test_custom_tables_default_location(self): expected_cmor_folder = os.path.join( os.path.dirname(esmvalcore.cmor.__file__), "tables", - "custom", + "cmip5-custom", ) - self.assertEqual(custom_info._cmor_folder, expected_cmor_folder) + assert custom_info.paths == (Path(expected_cmor_folder),) self.assertTrue(custom_info.tables["custom"]) self.assertTrue(custom_info.coords) def test_custom_tables_location(self): """Test constructor with custom tables location.""" cmor_path = os.path.dirname(os.path.realpath(esmvalcore.cmor.__file__)) - default_cmor_tables_path = os.path.join(cmor_path, "tables", "custom") + default_cmor_tables_path = os.path.join( + cmor_path, + "tables", + "cmip5-custom", + ) cmor_tables_path = os.path.join(cmor_path, "tables", "cmip5") cmor_tables_path = os.path.abspath(cmor_tables_path) custom_info = CustomInfo(cmor_tables_path) - self.assertEqual(custom_info._cmor_folder, default_cmor_tables_path) - self.assertEqual(custom_info._user_table_folder, cmor_tables_path) + assert custom_info.paths == ( + Path(default_cmor_tables_path), + Path(cmor_tables_path), + ) self.assertTrue(custom_info.tables["custom"]) self.assertTrue(custom_info.coords) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 3fae8cd8e6..ff8ec4fdc7 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,22 +1,20 @@ from __future__ import annotations import os -from pathlib import Path from typing import TYPE_CHECKING import iris import pytest -import esmvalcore.io.local from esmvalcore.io.local import ( + LocalDataSource, LocalFile, - _replace_tags, _select_files, ) -from esmvalcore.local import _select_drs if TYPE_CHECKING: from collections.abc import Callable, Iterator + from pathlib import Path from esmvalcore.typing import Facets, FacetValue @@ -36,11 +34,12 @@ def create_test_file(filename, tracking_id=None): def _get_files( # noqa: C901,PLR0912 + self: LocalDataSource, root_path: Path, facets: Facets, tracking_id: Iterator[int], suffix: str = "nc", -) -> tuple[list[LocalFile], list[Path]]: +) -> list[LocalFile]: """Return dummy files. Wildcards are only supported for `dataset` and `institute`; in this case @@ -55,34 +54,15 @@ def _get_files( # noqa: C901,PLR0912 else: all_facets = [facets] - # Globs without expanded facets - dir_template = _select_drs("input_dir", facets["project"], "default") # type: ignore[arg-type] - file_template = _select_drs("input_file", facets["project"], "default") # type: ignore[arg-type] - dir_globs = _replace_tags(dir_template, facets) - file_globs = _replace_tags(file_template, facets) - globs = sorted( - root_path / "input" / d / f for d in dir_globs for f in file_globs - ) + self.rootpath = root_path / "input" + facets = dict(facets) + if "original_short_name" in facets: + facets["short_name"] = facets["original_short_name"] files = [] for expanded_facets in all_facets: filenames = [] - dir_template = _select_drs( - "input_dir", - expanded_facets["project"], # type: ignore[arg-type] - "default", - ) - file_template = _select_drs( - "input_file", - expanded_facets["project"], # type: ignore[arg-type] - "default", - ) - - dir_globs = _replace_tags(dir_template, expanded_facets) - file_globs = _replace_tags(file_template, expanded_facets) - filename = str( - root_path / "input" / dir_globs[0] / Path(file_globs[0]).name, - ) + filename = str(self._get_glob_patterns(**expanded_facets)[0]) if filename.endswith("nc"): filename = f"{filename[:-2]}{suffix}" @@ -120,7 +100,7 @@ def _get_files( # noqa: C901,PLR0912 if "timerange" in facets: files = _select_files(files, facets["timerange"]) - return files, globs + return files def _tracking_ids(i=0): @@ -129,37 +109,28 @@ def _tracking_ids(i=0): i += 1 -def _get_find_files_func( +def _get_find_data_func( path: Path, suffix: str = "nc", -) -> Callable[ - ..., - tuple[list[LocalFile], list[Path]] | list[LocalFile], -]: +) -> Callable[..., list[LocalFile]]: tracking_id = _tracking_ids() - def find_files( - self: esmvalcore.io.local.LocalDataSource, - *, - debug: bool = False, + def find_data( + self: LocalDataSource, **facets: FacetValue, - ) -> tuple[list[LocalFile], list[Path]] | list[LocalFile]: - files, file_globs = _get_files(path, facets, tracking_id, suffix) - if debug: - return files, file_globs - return files + ) -> list[LocalFile]: + return _get_files(self, path, facets, tracking_id, suffix) - return find_files + return find_data @pytest.fixture -def patched_datafinder(tmp_path: Path, monkeypatch: pytest.MonkeyPath) -> None: - find_files = _get_find_files_func(tmp_path) - monkeypatch.setattr( - esmvalcore.io.local.LocalDataSource, - "find_data", - find_files, - ) +def patched_datafinder( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + find_data = _get_find_data_func(tmp_path) + monkeypatch.setattr(LocalDataSource, "find_data", find_data) @pytest.fixture @@ -167,12 +138,8 @@ def patched_datafinder_grib( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - find_files = _get_find_files_func(tmp_path, suffix="grib") - monkeypatch.setattr( - esmvalcore.io.local.LocalDataSource, - "find_data", - find_files, - ) + find_data = _get_find_data_func(tmp_path, suffix="grib") + monkeypatch.setattr(LocalDataSource, "find_data", find_data) @pytest.fixture @@ -191,25 +158,17 @@ def patched_failing_datafinder( """ tracking_id = _tracking_ids() - def find_files( - self: esmvalcore.io.local.LocalDataSource, - *, - debug: bool = False, + def find_data( + self: LocalDataSource, **facets: FacetValue, - ) -> tuple[list[LocalFile], list[Path]] | list[LocalFile]: - files, file_globs = _get_files(tmp_path, facets, tracking_id) + ) -> list[LocalFile]: + files = _get_files(self, tmp_path, facets, tracking_id) if facets["frequency"] == "fx": files = [] returned_files = [] for file in files: if not ("AAA" in file.name and "rsutcs" in file.name): returned_files.append(file) - if debug: - return returned_files, file_globs return returned_files - monkeypatch.setattr( - esmvalcore.io.local.LocalDataSource, - "find_data", - find_files, - ) + monkeypatch.setattr(LocalDataSource, "find_data", find_data) diff --git a/tests/integration/io/test_local.py b/tests/integration/io/test_local.py index 81a536f196..fc8e5ffcba 100644 --- a/tests/integration/io/test_local.py +++ b/tests/integration/io/test_local.py @@ -9,6 +9,7 @@ import pytest import yaml +import esmvalcore from esmvalcore.config import CFG from esmvalcore.io.local import ( LocalDataSource, @@ -66,8 +67,13 @@ def create_tree(path, filenames=None, symlinks=None): @pytest.mark.parametrize("cfg", CONFIG["get_output_file"]) -def test_get_output_file(cfg): +def test_get_output_file(monkeypatch, cfg): """Test getting output name for preprocessed files.""" + monkeypatch.setitem( + CFG, + "config_developer_file", + Path(esmvalcore.__path__[0], "config-developer.yml"), + ) output_file = _get_output_file(cfg["variable"], cfg["preproc_dir"]) expected = Path(cfg["output_file"]) assert output_file == expected @@ -95,6 +101,11 @@ def test_find_files(monkeypatch, root, cfg): pprint.pformat(cfg["variable"]), ) project = cfg["variable"]["project"] + monkeypatch.setitem( + CFG, + "config_developer_file", + Path(esmvalcore.__path__[0], "config-developer.yml"), + ) monkeypatch.setitem(CFG, "drs", {project: cfg["drs"]}) monkeypatch.setitem(CFG, "rootpath", {project: root}) create_tree( diff --git a/tests/integration/recipe/test_recipe.py b/tests/integration/recipe/test_recipe.py index b87a696387..7799ba522f 100644 --- a/tests/integration/recipe/test_recipe.py +++ b/tests/integration/recipe/test_recipe.py @@ -3294,7 +3294,7 @@ def test_bias_two_refs_with_mmm(tmp_path, patched_datafinder, session): additional_datasets: - {dataset: CanESM5, group: ref, reference_for_bias: true} - {dataset: CESM2, group: ref, reference_for_bias: true} - - {dataset: MPI-ESM-LR, group: notref} + - {dataset: MPI-ESM1-2-LR, group: notref} scripts: null """) @@ -3562,7 +3562,7 @@ def test_distance_metrics_two_refs_with_mmm( additional_datasets: - {dataset: CESM2, ensemble: r1i1p1f1, reference_for_metric: true} - {dataset: CESM2, ensemble: r2i1p1f1, reference_for_metric: true} - - {dataset: MPI-ESM-LR} + - {dataset: MPI-ESM1-2-LR} scripts: null """) @@ -3825,9 +3825,9 @@ def test_align_metadata_invalid_project(tmp_path, patched_datafinder, session): """) msg = ( "align_metadata failed: \"No CMOR tables available for project 'ZZZ'. " - "The following tables are available: custom, CMIP7, CMIP6, CMIP5, " - "CMIP3, OBS, OBS6, native6, obs4MIPs, ana4MIPs, EMAC, CORDEX, IPSLCM, " - 'ICON, CESM, ACCESS."' + "The following tables are available: CMIP7, CMIP6, CMIP5, CMIP3, " + "CORDEX, obs4MIPs, ana4MIPs, native6, ACCESS, CESM, EMAC, ICON, IPSLCM, " + 'OBS6, OBS."' ) with pytest.raises(RecipeError) as exc: get_recipe(tmp_path, content, session) diff --git a/tests/unit/config/test_config.py b/tests/unit/config/test_config.py index 035ccbd7f2..0b7aa40d28 100644 --- a/tests/unit/config/test_config.py +++ b/tests/unit/config/test_config.py @@ -161,14 +161,14 @@ def test_get_project_config(mocker): def test_load_default_config(cfg_default, monkeypatch): """Test that the default configuration can be loaded.""" - project_cfg = {} - monkeypatch.setattr(_config, "CFG", project_cfg) root_path = importlib_files("esmvalcore") - default_dev_file = root_path / "config-developer.yml" config_dir = root_path / "config" / "configurations" / "defaults" default_project_settings = dask.config.collect( paths=[str(p) for p in config_dir.glob("extra_facets_*.yml")] - + [str(config_dir / "preprocessor_filename_template.yml")], + + [ + str(config_dir / "preprocessor_filename_template.yml"), + str(config_dir / "cmor_tables.yml"), + ], env={}, )["projects"] @@ -178,7 +178,7 @@ def test_load_default_config(cfg_default, monkeypatch): "auxiliary_data_dir": Path.home() / "auxiliary_data", "check_level": CheckLevels.DEFAULT, "compress_netcdf": False, - "config_developer_file": default_dev_file, + "config_developer_file": None, "dask": { "profiles": { "local_threaded": { @@ -261,9 +261,6 @@ def test_load_default_config(cfg_default, monkeypatch): assert getattr(session, path + "_dir") == session.session_dir / path assert session.plot_dir == session.session_dir / "plots" - # Check that projects were configured - assert project_cfg - def test_rootpath_obs4mips_case_correction(monkeypatch): """Test that the name of the obs4MIPs project is correct in rootpath.""" @@ -328,8 +325,13 @@ def test_get_ignored_warnings_none(project, step): assert get_ignored_warnings(project, step) is None -def test_get_ignored_warnings_emac(): +def test_get_ignored_warnings_emac(monkeypatch: pytest.MonkeyPatch) -> None: """Test ``get_ignored_warnings``.""" + monkeypatch.setitem( + CFG, + "config_developer_file", + Path(esmvalcore.__path__[0], "config-developer.yml"), + ) ignored_warnings = get_ignored_warnings("EMAC", "load") assert isinstance(ignored_warnings, list) assert ignored_warnings diff --git a/tests/unit/config/test_config_validator.py b/tests/unit/config/test_config_validator.py index eed9b19bd5..44e92e86cc 100644 --- a/tests/unit/config/test_config_validator.py +++ b/tests/unit/config/test_config_validator.py @@ -311,13 +311,13 @@ def test_handle_deprecation(remove_version): def test_validate_config_developer_none(): """Test ``validate_config_developer``.""" path = validate_config_developer(None) - assert path == Path(esmvalcore.__file__).parent / "config-developer.yml" + assert path is None def test_validate_config_developer(tmp_path): """Test ``validate_config_developer``.""" custom_table_path = ( - Path(esmvalcore.__file__).parent / "cmor" / "tables" / "custom" + Path(esmvalcore.__file__).parent / "cmor" / "tables" / "cmip5-custom" ) cfg_dev = { "custom": {"cmor_path": custom_table_path}, diff --git a/tests/unit/config/test_data_sources.py b/tests/unit/config/test_data_sources.py index b1e08f69af..fce0b3a625 100644 --- a/tests/unit/config/test_data_sources.py +++ b/tests/unit/config/test_data_sources.py @@ -1,5 +1,6 @@ from __future__ import annotations +from pathlib import Path from typing import TYPE_CHECKING import pytest @@ -37,6 +38,11 @@ def test_load_legacy_data_sources( session["projects"][project].pop("data", None) session["search_esgf"] = search_esgf session["download_dir"] = "~/climate_data" + monkeypatch.setitem( + esmvalcore.local.CFG, + "config_developer_file", + Path(esmvalcore.__path__[0], "config-developer.yml"), + ) monkeypatch.setitem( esmvalcore.local.CFG, "rootpath", diff --git a/tests/unit/io/local/test_get_data_sources.py b/tests/unit/io/local/test_get_data_sources.py index 8f19709b8a..c086947a57 100644 --- a/tests/unit/io/local/test_get_data_sources.py +++ b/tests/unit/io/local/test_get_data_sources.py @@ -5,8 +5,8 @@ import pytest +import esmvalcore from esmvalcore.config import CFG -from esmvalcore.config._config_validators import validate_config_developer from esmvalcore.io.local import LocalDataSource from esmvalcore.local import DataSource, _get_data_sources @@ -33,7 +33,11 @@ ) def test_get_data_sources(monkeypatch, rootpath_drs): # Make sure that default config-developer file is used - validate_config_developer(None) + monkeypatch.setitem( + CFG, + "config_developer_file", + Path(esmvalcore.__path__[0], "config-developer.yml"), + ) rootpath, drs = rootpath_drs monkeypatch.setitem(CFG, "rootpath", rootpath) @@ -48,7 +52,11 @@ def test_get_data_sources(monkeypatch, rootpath_drs): def test_get_data_sources_nodefault(monkeypatch): # Make sure that default config-developer file is used - validate_config_developer(None) + monkeypatch.setitem( + CFG, + "config_developer_file", + Path(esmvalcore.__path__[0], "config-developer.yml"), + ) monkeypatch.setitem( CFG, diff --git a/tests/unit/preprocessor/test_configuration.py b/tests/unit/preprocessor/test_configuration.py index 454b9514a2..539f237873 100644 --- a/tests/unit/preprocessor/test_configuration.py +++ b/tests/unit/preprocessor/test_configuration.py @@ -2,10 +2,13 @@ from __future__ import annotations +from pathlib import Path from typing import TYPE_CHECKING import pytest +import esmvalcore +import esmvalcore.config from esmvalcore.dataset import Dataset from esmvalcore.exceptions import RecipeError from esmvalcore.preprocessor import ( @@ -147,9 +150,15 @@ def test_get_preprocessor_filename_default( def test_get_preprocessor_filename_falls_back_to_config_developer( + monkeypatch: pytest.MonkeyPatch, session: Session, ) -> None: """Test the function `_get_preprocessor_filename`.""" + monkeypatch.setitem( + esmvalcore.config.CFG, + "config_developer_file", + Path(esmvalcore.__path__[0], "config-developer.yml"), + ) session["projects"]["CMIP6"].pop("preprocessor_filename_template") dataset = Dataset( project="CMIP6", From aaaeb9e15c3caeee116d257232bea0ac729b6742 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Mon, 19 Jan 2026 17:07:30 +0100 Subject: [PATCH 02/47] Fix another test --- tests/integration/io/test_local.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/integration/io/test_local.py b/tests/integration/io/test_local.py index fc8e5ffcba..b15f86ac75 100644 --- a/tests/integration/io/test_local.py +++ b/tests/integration/io/test_local.py @@ -187,7 +187,12 @@ def test_find_data(root, cfg): assert str(pattern) in data_source.debug_info -def test_select_invalid_drs_structure(): +def test_select_invalid_drs_structure(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setitem( + CFG, + "config_developer_file", + Path(esmvalcore.__path__[0], "config-developer.yml"), + ) msg = ( r"drs _INVALID_STRUCTURE_ for CMIP6 project not specified in " r"config-developer file" From ea4aa6a6eedb36d3f22b460d6607f73f8c620cf3 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Fri, 23 Jan 2026 17:49:29 +0100 Subject: [PATCH 03/47] Add docstrings and type hints --- esmvalcore/cmor/_utils.py | 2 +- esmvalcore/cmor/table.py | 270 +++++++++++++++++------- esmvalcore/config/_config.py | 1 + esmvalcore/config/_config_validators.py | 25 +++ 4 files changed, 217 insertions(+), 81 deletions(-) diff --git a/esmvalcore/cmor/_utils.py b/esmvalcore/cmor/_utils.py index da8eddd759..f939caadf3 100644 --- a/esmvalcore/cmor/_utils.py +++ b/esmvalcore/cmor/_utils.py @@ -163,7 +163,7 @@ def _get_new_generic_level_coord( New generic level coordinate. """ - new_coord = generic_level_coord.generic_lev_coords[new_coord_name] + new_coord = generic_level_coord.generic_lev_coords[new_coord_name] # type: ignore[index] # Is this a bug? new_coord.generic_level = True new_coord.generic_lev_coords = var_info.coordinates[ generic_level_coord_name diff --git a/esmvalcore/cmor/table.py b/esmvalcore/cmor/table.py index f19978ce51..f3610eb49a 100644 --- a/esmvalcore/cmor/table.py +++ b/esmvalcore/cmor/table.py @@ -15,7 +15,7 @@ from collections import Counter from functools import lru_cache, total_ordering from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Self import yaml @@ -43,7 +43,7 @@ def _get_institutes(project: str, dataset: str) -> list[str]: - """Return the institutes given the dataset name in CMIP6.""" + """Return the institutes from the controlled vocabulary given the dataset name.""" try: return CMOR_TABLES[project].institutes[dataset] # type: ignore[attr-defined] except (KeyError, AttributeError): @@ -54,7 +54,7 @@ def _get_activity( project: str, exp: str | list[str], ) -> str | list[str] | None: - """Return the activity given the experiment name in CMIP6.""" + """Return the activity from the controlled vocabulary given the experiment name.""" try: if isinstance(exp, list): return [CMOR_TABLES[project].activities[value][0] for value in exp] # type: ignore[attr-defined] @@ -206,6 +206,12 @@ def load_cmor_tables(cfg: Config) -> None: def read_cmor_tables(cfg_developer: Path | None = None) -> None: """Read cmor tables required in the configuration. + .. deprecated:: 2.14.0 + + The config-developer.yml file based configuration is deprecated and + will no longer be supported in ESMValCore v2.16.0. Please use + :func:`~esmvalcore.cmor.table.load_cmor_tables` instead of this function. + Parameters ---------- cfg_developer: @@ -374,34 +380,34 @@ def get_tables( class InfoBase: - """Base class for all table info classes. - - This uses CMOR 3 json format + """Base class for all CMOR table info classes. Parameters ---------- - cmor_tables_path: - The path to a directory with subdirectory "Tables" where the CMOR tables - are located. - default: Default table to look variables on if not found. + .. deprecated:: 2.14.0 + + The ``default`` parameter is deprecated and will be removed in + ESMValCore v2.16.0. Please use the ``paths`` parameter instead + to aggregate multiple tables. + alt_names: List of known alternative names for variables. If no value is provided, - the default values from the file variable_alt_names.yml will be used. + the default values from the installed copy of + `variable_alt_names.yml `_ + will be used. strict: bool If False, will look for a variable in other tables if it can not be found in the requested one. - default_table_prefix: - If the table_id contains a prefix, it can be specified here. - paths: A list of paths to CMOR tables. If the path is relative and exists in - the ``tables`` directory in :mod:`esmvalcore.cmor`, the version of the - tables shipped with ESMValCore will be used. + the installed copy of the + `esmvalcore/cmor/tables `_ + directory it will be used. """ def __init__( @@ -420,6 +426,7 @@ def __init__( else p for p in paths ) + """A list of paths to CMOR tables.""" for path in self.paths: if not path.is_dir(): raise NotADirectoryError(path) @@ -431,10 +438,24 @@ def __init__( alt_names_path.read_text(encoding="utf-8"), ) self.alt_names = alt_names + """List of known alternative names for variables.""" self.coords: dict[str, CoordinateInfo] = {} + """The coordinates defined in these tables.""" self.default = default + """ + Default table to look variables on if not found. + + .. deprecated:: 2.14.0 + + The ``default`` attribute is deprecated and will be removed in + ESMValCore v2.16.0. + """ self.strict = strict + """If False, will look for a variable in other tables if it can not be + found in the requested one. + """ self.tables: dict[str, TableInfo] = {} + """A mapping from table names to :class:`TableInfo` objects.""" def get_table(self, table: str) -> TableInfo | None: """Search and return the table info. @@ -560,9 +581,9 @@ def _look_all_tables(self, alt_names): class CMIP6Info(InfoBase): - """Class to read CMIP6-like data request. + """Class to read CMIP6-like CMOR tables. - This uses CMOR 3 json format + This class reads CMOR 3 json format tables. Parameters ---------- @@ -570,12 +591,24 @@ class CMIP6Info(InfoBase): The path to a directory with subdirectory "Tables" where the CMOR tables are located. + .. deprecated:: 2.14.0 + + The ``cmor_tables_path`` parameter is deprecated and will be removed in + ESMValCore v2.16.0. Please use the ``paths`` parameter instead. + default: Default table to look variables on if not found. + .. deprecated:: 2.14.0 + + The ``default`` parameter is deprecated and will be removed in + ESMValCore v2.16.0. Please use the ``paths`` parameter instead + to aggregate multiple tables. + alt_names: List of known alternative names for variables. If no value is provided, - the default values from the file variable_alt_names.yml will be used. + the default values from the installed copy of + `variable_alt_names.yml`_ will be used. strict: bool If False, will look for a variable in other tables if it can not be @@ -584,11 +617,15 @@ class CMIP6Info(InfoBase): default_table_prefix: If the table_id contains a prefix, it can be specified here. + .. deprecated:: 2.14.0 + + The ``default_table_prefix`` parameter is deprecated and will be removed in + ESMValCore v2.16.0. + paths: A list of paths to CMOR tables. If the path is relative and exists in - the installed copy of the - `esmvalcore/cmor/tables `__ - directory it will be used. + the installed copy of the `esmvalcore/cmor/tables`_ directory it will + be used. """ def __init__( @@ -613,9 +650,19 @@ def __init__( super().__init__(default, alt_names, strict, paths=paths) self.default_table_prefix = default_table_prefix + """ + If the table_id contains a prefix, it can be specified here. + + .. deprecated:: 2.14.0 + + The ``default_table_prefix`` attribute is deprecated and will be + removed in ESMValCore v2.16.0. + """ self.var_to_freq: dict[str, dict[str, str]] = {} self.activities: dict[str, list[str]] = {} + """A mapping from ``exp`` to ``activity`` from the controlled vocabulary.""" self.institutes: dict[str, list[str]] = {} + """A mapping from ``dataset`` to ``institute`` from the controlled vocabulary.""" for path in self.paths: if not any(path.glob("*.json")): @@ -665,7 +712,7 @@ def _load_table(self, json_file): self.var_to_freq[table.name] = {} for var_name, var_data in raw_data["variable_entry"].items(): - var = VariableInfo("CMIP6", var_name) + var = VariableInfo("CMIP6", name=var_name) var.read_json(var_data, table.frequency) self._assign_dimensions(var, generic_levels) table[var_name] = var @@ -731,17 +778,17 @@ def _load_controlled_vocabulary(self, path: Path) -> None: except (KeyError, AttributeError): pass - def get_table(self, table): + def get_table(self, table: str) -> TableInfo | None: """Search and return the table info. Parameters ---------- - table: str + table: Table name Returns ------- - TableInfo + : Return the TableInfo object for the requested table if found, returns None if not """ @@ -758,22 +805,14 @@ def _is_table(table_data): class Obs4MIPsInfo(CMIP6Info): - """Class to read obs4MIPs-like data request. - - This uses CMOR 3 json format + """Class to read obs4MIPs-like CMOR tables. Parameters ---------- - cmor_tables_path: - The path to a directory with subdirectory "Tables" where the CMOR tables - are located. - - default: - Default table to look variables on if not found. - alt_names: List of known alternative names for variables. If no value is provided, - the default values from the file variable_alt_names.yml will be used. + the default values from the installed copy of + `variable_alt_names.yml`_ will be used. strict: bool If False, will look for a variable in other tables if it can not be @@ -781,22 +820,17 @@ class Obs4MIPsInfo(CMIP6Info): paths: A list of paths to CMOR tables. If the path is relative and exists in - the installed copy of the - `esmvalcore/cmor/tables `__ - directory it will be used. + the ``tables`` directory in :mod:`esmvalcore.cmor`, the version of the + tables shipped with ESMValCore will be used. """ def __init__( self, - cmor_tables_path: str | None = None, - default: CustomInfo | None = None, alt_names: list[list[str]] | None = None, strict: bool = True, paths: Iterable[Path] = (), ) -> None: super().__init__( - cmor_tables_path=cmor_tables_path, - default=default, alt_names=alt_names, strict=strict, paths=paths, @@ -817,8 +851,11 @@ def __init__(self, *args, **kwargs): """Create a new TableInfo object for storing VariableInfo objects.""" super().__init__(*args, **kwargs) self.name = "" + """Table name.""" self.frequency = "" + """Table frequency (if defined).""" self.realm = "" + """Table realm (if defined).""" def __eq__(self, other): return (self.name, self.frequency, self.realm) == ( @@ -892,18 +929,38 @@ def _read_json_list_variable(self, parameter): class VariableInfo(JsonInfo): """Class to read and store variable information.""" - def __init__(self, table_type, short_name): + def __init__( + self, + table_type: str = "", + short_name: str = "", + name: str = "", + ) -> None: """Class to read and store variable information. Parameters ---------- - short_name: str + table_type: + Type of table (e.g., CMIP5, CMIP6). + + .. deprecated:: 2.14.0 + + The ``table_type`` parameter is deprecated and will be removed + in ESMValCore v2.16.0. + short_name: Variable's short name. + + .. deprecated:: 2.14.0 + + The ``short_name`` parameter is deprecated and will be removed + in ESMValCore v2.16.0. + name: + Name of the variable entry in the CMOR table. """ super().__init__() - self.name = short_name + self.name = name + """Name of the variable entry in the CMOR table.""" self.table_type = table_type - self.modeling_realm = [] + self.modeling_realm: list[str] = [] """Modeling realm""" self.short_name = short_name """Short name""" @@ -922,9 +979,9 @@ def __init__(self, table_type, short_name): self.positive = "" """Increasing direction""" - self.dimensions = [] + self.dimensions: list[str] = [] """List of dimensions""" - self.coordinates = {} + self.coordinates: dict[str, CoordinateInfo] = {} """Coordinates This is a dict with the names of the dimensions as keys and @@ -936,7 +993,7 @@ def __init__(self, table_type, short_name): def __repr__(self) -> str: return f"{self.__class__.__name__}(name={self.name})" - def copy(self): + def copy(self) -> Self: """Return a shallow copy of VariableInfo. Returns @@ -946,17 +1003,17 @@ def copy(self): """ return copy.copy(self) - def read_json(self, json_data, default_freq): + def read_json(self, json_data: dict, default_freq: str) -> None: """Read variable information from json. Non-present options will be set to empty Parameters ---------- - json_data: dict + json_data: Dictionary created by the json reader containing variable information. - default_freq: str + default_freq: Default frequency to use if it is not defined at variable level. """ self._json_data = json_data @@ -992,12 +1049,12 @@ def has_coord_with_standard_name(self, standard_name: str) -> bool: Parameters ---------- - standard_name: str + standard_name: Standard name to be checked. Returns ------- - bool + : `True` if there is at least one coordinate with the given `standard_name`, `False` if not. @@ -1011,18 +1068,19 @@ def has_coord_with_standard_name(self, standard_name: str) -> bool: class CoordinateInfo(JsonInfo): """Class to read and store coordinate information.""" - def __init__(self, name): + def __init__(self, name: str) -> None: """Class to read and store coordinate information. Parameters ---------- - name: str + name: coordinate's name """ super().__init__() self.name = name + """Name of the coordinate entry in the CMOR table.""" self.generic_level = False - self.generic_lev_coords = {} + self.generic_lev_coords: dict[str, CoordinateInfo] = {} self.axis = "" """Axis""" @@ -1044,7 +1102,7 @@ def __init__(self, name): """Units""" self.stored_direction = "" """Direction in which the coordinate increases""" - self.requested = [] + self.requested: list[str] = [] """Values requested""" self.valid_min = "" """Minimum allowed value""" @@ -1071,7 +1129,7 @@ def read_json(self, json_data): self.axis = self._read_json_variable("axis") self.value = self._read_json_variable("value") self.out_name = self._read_json_variable("out_name") - self.var_name = self._read_json_variable("var_name") + self.var_name = self._read_json_variable("out_name") self.standard_name = self._read_json_variable("standard_name") self.long_name = self._read_json_variable("long_name") self.units = self._read_json_variable("units") @@ -1084,7 +1142,9 @@ def read_json(self, json_data): class CMIP5Info(InfoBase): - """Class to read CMIP5-like data request. + """Class to read CMIP5-like CMOR tables. + + This class reads CMOR 2 format tables. Parameters ---------- @@ -1092,22 +1152,42 @@ class CMIP5Info(InfoBase): The path to a directory with subdirectory "Tables" where the CMOR tables are located. + .. deprecated:: 2.14.0 + + The ``cmor_tables_path`` parameter is deprecated and will be removed in + ESMValCore v2.16.0. Please use the ``paths`` parameter instead. + default: Default table to look variables on if not found. + .. deprecated:: 2.14.0 + + The ``default`` parameter is deprecated and will be removed in + ESMValCore v2.16.0. Please use the ``paths`` parameter instead + to aggregate multiple tables. + alt_names: List of known alternative names for variables. If no value is provided, - the default values from the file variable_alt_names.yml will be used. + the default values from the installed copy of + `variable_alt_names.yml`_ will be used. strict: bool If False, will look for a variable in other tables if it can not be found in the requested one. + default_table_prefix: + If the table_id contains a prefix, it can be specified here. + + .. deprecated:: 2.14.0 + + The ``default_table_prefix`` parameter is deprecated and will be removed in + ESMValCore v2.16.0. + paths: A list of paths to CMOR tables. If the path is relative and exists in - the installed copy of the - `esmvalcore/cmor/tables `__ - directory it will be used. + the installed copy of the `esmvalcore/cmor/tables`_ directory it will + be used. + """ def __init__( @@ -1224,8 +1304,8 @@ def _read_coordinate(self, value): setattr(coord, key, value) return coord - def _read_variable(self, short_name, frequency): - var = VariableInfo("CMIP5", short_name) + def _read_variable(self, entry_name, frequency): + var = VariableInfo(table_type="CMIP5", name=entry_name) var.frequency = frequency while self._read_line(): key, value = self._last_line_read @@ -1241,17 +1321,17 @@ def _read_variable(self, short_name, frequency): var.coordinates[dim] = self.coords[dim] return var - def get_table(self, table): + def get_table(self, table: str) -> TableInfo | None: """Search and return the table info. Parameters ---------- - table: str + table: Table name Returns ------- - TableInfo + : Return the TableInfo object for the requested table if found, returns None if not """ @@ -1259,7 +1339,7 @@ def get_table(self, table): class CMIP3Info(CMIP5Info): - """Class to read CMIP3-like data request. + """Class to read CMIP3-like CMOR tables. Parameters ---------- @@ -1267,22 +1347,42 @@ class CMIP3Info(CMIP5Info): The path to a directory with subdirectory "Tables" where the CMOR tables are located. + .. deprecated:: 2.14.0 + + The ``cmor_tables_path`` parameter is deprecated and will be removed in + ESMValCore v2.16.0. Please use the ``paths`` parameter instead. + default: Default table to look variables on if not found. + .. deprecated:: 2.14.0 + + The ``default`` parameter is deprecated and will be removed in + ESMValCore v2.16.0. Please use the ``paths`` parameter instead + to aggregate multiple tables. + alt_names: List of known alternative names for variables. If no value is provided, - the default values from the file variable_alt_names.yml will be used. + the default values from the installed copy of + `variable_alt_names.yml`_ will be used. strict: bool If False, will look for a variable in other tables if it can not be found in the requested one. + default_table_prefix: + If the table_id contains a prefix, it can be specified here. + + .. deprecated:: 2.14.0 + + The ``default_table_prefix`` parameter is deprecated and will be removed in + ESMValCore v2.16.0. + paths: A list of paths to CMOR tables. If the path is relative and exists in - the installed copy of the - `esmvalcore/cmor/tables `__ - directory it will be used. + the installed copy of the `esmvalcore/cmor/tables`_ directory it will + be used. + """ def _read_table_file(self, table_file: str) -> TableInfo: @@ -1300,8 +1400,8 @@ def _read_coordinate(self, value): coord.var_name = coord.name return coord - def _read_variable(self, short_name, frequency): - var = super()._read_variable(short_name, frequency) + def _read_variable(self, entry_name, frequency): + var = super()._read_variable(entry_name, frequency) var.frequency = "" var.modeling_realm = [] return var @@ -1310,6 +1410,11 @@ def _read_variable(self, short_name, frequency): class CustomInfo(CMIP5Info): """Class to read custom var info for ESMVal. + .. deprecated:: 2.14.0 + + This class is deprecated and will be removed in ESMValCore v2.16.0. + Please use :class:`~esmvalcore.cmor.tables.table.CMIP5Info` instead. + Parameters ---------- cmor_tables_path: @@ -1426,7 +1531,10 @@ def _read_table_file(self, table_file: str) -> TableInfo: class NoInfo(InfoBase): - """Table that can be used for projects that do not have a CMOR table.""" + """Table that can be used for projects that do not provide a CMOR table.""" + + def __init__(self) -> None: + pass def get_variable( self, @@ -1456,7 +1564,9 @@ def get_variable( otherwise. """ - return VariableInfo(table_type="No table", short_name=short_name) + vardef = VariableInfo(name=short_name) + vardef.short_name = short_name + return vardef # Load the default tables on initializing the module. diff --git a/esmvalcore/config/_config.py b/esmvalcore/config/_config.py index 9d9ff9d6b5..aaf0ad47c8 100644 --- a/esmvalcore/config/_config.py +++ b/esmvalcore/config/_config.py @@ -1,4 +1,5 @@ """Functions dealing with config-developer.yml and extra facets.""" +# TODO: remove this module in v2.16.0 from __future__ import annotations diff --git a/esmvalcore/config/_config_validators.py b/esmvalcore/config/_config_validators.py index 33a94ff36f..6324b374d9 100644 --- a/esmvalcore/config/_config_validators.py +++ b/esmvalcore/config/_config_validators.py @@ -630,6 +630,30 @@ def deprecate_search_esgf( ) +def deprecate_config_developer_file( + validated_config: ValidatedConfig, # noqa: ARG001 + value: str | Path, # noqa: ARG001 + validated_value: str | Path, # noqa: ARG001 +) -> None: + """Deprecate ``config_developer_file`` option. + + Parameters + ---------- + validated_config: + ``ValidatedConfig`` instance which will be modified in place. + value: + Raw input value for ``config_file`` option. + validated_value: + Validated value for ``config_file`` option. + + """ + more_info = ( + " Please configure data sources, cmor tables, and preprocessor " + "filename templates under `projects` instead." + ) + _handle_deprecation("config_developer_file", "2.14.0", "2.16.0", more_info) + + # Example usage: see removed files in # https://github.com/ESMValGroup/ESMValCore/pull/2213 _deprecators: dict[str, Callable] = { @@ -638,6 +662,7 @@ def deprecate_search_esgf( "rootpath": deprecate_rootpath, # TODO: remove in v2.16.0 "download_dir": deprecate_download_dir, # TODO: remove in v2.16.0 "search_esgf": deprecate_search_esgf, # TODO: remove in v2.16.0 + "config_developer_file": deprecate_config_developer_file, # TODO: remove in v2.16.0 } From 64f365bb211c3e2c1bc54040b75c11356fe48c87 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Fri, 23 Jan 2026 18:03:00 +0100 Subject: [PATCH 04/47] Gracefully handle missing out_name in CMIP5-style CMOR tables and update default config --- esmvalcore/cmor/table.py | 3 +++ esmvalcore/config/_config_validators.py | 1 + esmvalcore/config/configurations/defaults/config-user.yml | 5 ----- tests/unit/cmor/test_table.py | 4 +++- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/esmvalcore/cmor/table.py b/esmvalcore/cmor/table.py index f3610eb49a..7a083ff58d 100644 --- a/esmvalcore/cmor/table.py +++ b/esmvalcore/cmor/table.py @@ -1317,6 +1317,9 @@ def _read_variable(self, entry_name, frequency): setattr(var, key, value) elif key == "out_name": var.short_name = value + if not var.short_name: + # Some of our custom CMIP5 table entries are missing the `out_name` field. + var.short_name = var.name for dim in var.dimensions: var.coordinates[dim] = self.coords[dim] return var diff --git a/esmvalcore/config/_config_validators.py b/esmvalcore/config/_config_validators.py index 6324b374d9..666400f764 100644 --- a/esmvalcore/config/_config_validators.py +++ b/esmvalcore/config/_config_validators.py @@ -671,4 +671,5 @@ def deprecate_config_developer_file( # https://github.com/ESMValGroup/ESMValCore/pull/2213 _deprecated_options_defaults: dict[str, Any] = { "extra_facets_dir": [], # TODO: remove in v2.15.0 + "config_developer_file": None, # TODO: remove in v2.16.0 } diff --git a/esmvalcore/config/configurations/defaults/config-user.yml b/esmvalcore/config/configurations/defaults/config-user.yml index a8b959a70e..bfc7a157c5 100644 --- a/esmvalcore/config/configurations/defaults/config-user.yml +++ b/esmvalcore/config/configurations/defaults/config-user.yml @@ -54,11 +54,6 @@ compress_netcdf: false # step. These files are numbered according to the preprocessing order. save_intermediary_cubes: false -# Path to custom ``config-developer.yml`` file -# This can be used to customise project configurations. See -# ``config-developer.yml`` for an example. Set to ``null`` to use the default. -config_developer_file: null - # Use a profiling tool for the diagnostic run --- [false]/true # A profiler tells you which functions in your code take most time to run. # Only available for Python diagnostics. diff --git a/tests/unit/cmor/test_table.py b/tests/unit/cmor/test_table.py index 155933d958..2a12d18599 100644 --- a/tests/unit/cmor/test_table.py +++ b/tests/unit/cmor/test_table.py @@ -117,8 +117,10 @@ def test_read_standard_name(self): def test_read_var_name(self): """Test var_name.""" + # There does not appear to be a "var_name" field in any of the CMOR + # tables. Could the var_name attribute be removed? info = CoordinateInfo("var") - info.read_json({"var_name": self.value}) + info.read_json({"out_name": self.value}) self.assertEqual(info.var_name, self.value) def test_read_out_name(self): From d676fbd570e756aa4f9a88ff9111dc2a49e17ab3 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Mon, 26 Jan 2026 12:16:17 +0100 Subject: [PATCH 05/47] Remove config_developer_file from default configuration --- esmvalcore/config/_config_validators.py | 1 - tests/integration/io/test_local.py | 5 +++++ tests/unit/config/test_config.py | 1 - 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/esmvalcore/config/_config_validators.py b/esmvalcore/config/_config_validators.py index 666400f764..6324b374d9 100644 --- a/esmvalcore/config/_config_validators.py +++ b/esmvalcore/config/_config_validators.py @@ -671,5 +671,4 @@ def deprecate_config_developer_file( # https://github.com/ESMValGroup/ESMValCore/pull/2213 _deprecated_options_defaults: dict[str, Any] = { "extra_facets_dir": [], # TODO: remove in v2.15.0 - "config_developer_file": None, # TODO: remove in v2.16.0 } diff --git a/tests/integration/io/test_local.py b/tests/integration/io/test_local.py index b15f86ac75..6818bf85ac 100644 --- a/tests/integration/io/test_local.py +++ b/tests/integration/io/test_local.py @@ -132,6 +132,11 @@ def test_find_files_with_facets(monkeypatch, root): break project = cfg["variable"]["project"] + monkeypatch.setitem( + CFG, + "config_developer_file", + Path(esmvalcore.__path__[0], "config-developer.yml"), + ) monkeypatch.setitem(CFG, "drs", {project: cfg["drs"]}) monkeypatch.setitem(CFG, "rootpath", {project: root}) diff --git a/tests/unit/config/test_config.py b/tests/unit/config/test_config.py index 0b7aa40d28..4c8aed9775 100644 --- a/tests/unit/config/test_config.py +++ b/tests/unit/config/test_config.py @@ -178,7 +178,6 @@ def test_load_default_config(cfg_default, monkeypatch): "auxiliary_data_dir": Path.home() / "auxiliary_data", "check_level": CheckLevels.DEFAULT, "compress_netcdf": False, - "config_developer_file": None, "dask": { "profiles": { "local_threaded": { From d0090d48f00daa4b9018a98325ab69f766074b2d Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Mon, 26 Jan 2026 12:27:25 +0100 Subject: [PATCH 06/47] Use explicit configuration instead of YAML anchors --- .../configurations/defaults/cmor_tables.yml | 56 +++++++++++++++---- 1 file changed, 44 insertions(+), 12 deletions(-) diff --git a/esmvalcore/config/configurations/defaults/cmor_tables.yml b/esmvalcore/config/configurations/defaults/cmor_tables.yml index 06ec45c667..5da64dbf6c 100644 --- a/esmvalcore/config/configurations/defaults/cmor_tables.yml +++ b/esmvalcore/config/configurations/defaults/cmor_tables.yml @@ -8,13 +8,13 @@ projects: - cmip7/tables - cmip6-custom CMIP6: - cmor_table: &cmip6 + cmor_table: type: esmvalcore.cmor.table.CMIP6Info paths: - cmip6/Tables - cmip6-custom CMIP5: - cmor_table: &cmip5 + cmor_table: type: esmvalcore.cmor.table.CMIP5Info paths: - cmip5/Tables @@ -43,35 +43,67 @@ projects: # CMOR tables are available at https://github.com/PCMDI/ana4MIPs-cmor-tables/ # but it is not clear how well these match the data. cmor_table: - <<: *cmip5 + type: esmvalcore.cmor.table.CMIP5Info + paths: + - cmip5/Tables + - cmip5-custom strict: false # Observational and reanalysis data that can be read in its native format by ESMValCore. native6: - cmor_table: &native6 - <<: *cmip6 + cmor_table: + type: esmvalcore.cmor.table.CMIP6Info + paths: + - cmip6/Tables + - cmip6-custom strict: false # Data from selected climate models that can be read in its native format by ESMValCore. ACCESS: cmor_table: - <<: *native6 + type: esmvalcore.cmor.table.CMIP6Info + paths: + - cmip6/Tables + - cmip6-custom + strict: false CESM: cmor_table: - <<: *native6 + type: esmvalcore.cmor.table.CMIP6Info + paths: + - cmip6/Tables + - cmip6-custom + strict: false EMAC: cmor_table: - <<: *native6 + type: esmvalcore.cmor.table.CMIP6Info + paths: + - cmip6/Tables + - cmip6-custom + strict: false ICON: cmor_table: - <<: *native6 + type: esmvalcore.cmor.table.CMIP6Info + paths: + - cmip6/Tables + - cmip6-custom + strict: false IPSLCM: cmor_table: - <<: *native6 + type: esmvalcore.cmor.table.CMIP6Info + paths: + - cmip6/Tables + - cmip6-custom + strict: false # Data that has been CMORized by ESMValTool OBS6: cmor_table: - <<: *cmip6 + type: esmvalcore.cmor.table.CMIP6Info + paths: + - cmip6/Tables + - cmip6-custom strict: false OBS: cmor_table: - <<: *cmip5 + type: esmvalcore.cmor.table.CMIP5Info + paths: + - cmip5/Tables + - cmip5-custom strict: false From 33ce271cfbed16f0b86b07013b3700ccb4421538 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Wed, 28 Jan 2026 17:08:17 +0100 Subject: [PATCH 07/47] Small improvements --- doc/recipe/preprocessor.rst | 4 +++ esmvalcore/_recipe/check.py | 8 ++++- esmvalcore/cmor/table.py | 47 +++++++++++++++++++------ esmvalcore/config/_config_validators.py | 4 +-- esmvalcore/preprocessor/_other.py | 33 ++++++++++++++--- 5 files changed, 77 insertions(+), 19 deletions(-) diff --git a/doc/recipe/preprocessor.rst b/doc/recipe/preprocessor.rst index fd56ffa20b..a708d3aacd 100644 --- a/doc/recipe/preprocessor.rst +++ b/doc/recipe/preprocessor.rst @@ -2945,6 +2945,10 @@ recipe: target metadata is read. If not given, use the short names of the corresponding variables defined in the recipe. +* ``target_branding_suffix`` (:obj:`str`; optional): Variable branding suffix from which + target metadata is read. + If not given, use the branding suffixes of the corresponding variables defined in + the recipe. * ``strict`` (:obj:`str`; optional, default: ``True``): If ``True``, raise an error if desired metadata cannot be read for variable ``target_short_name`` of MIP table ``target_mip`` and project ``target_project``. diff --git a/esmvalcore/_recipe/check.py b/esmvalcore/_recipe/check.py index 1aae4b6aef..bb935cbc0a 100644 --- a/esmvalcore/_recipe/check.py +++ b/esmvalcore/_recipe/check.py @@ -48,6 +48,7 @@ def align_metadata(step_settings: dict[str, Any]) -> None: project = step_settings.get("target_project") mip = step_settings.get("target_mip") short_name = step_settings.get("target_short_name") + branding_suffix = step_settings.get("target_branding_suffix") strict = step_settings.get("strict", True) # Any missing arguments will be reported later @@ -55,7 +56,12 @@ def align_metadata(step_settings: dict[str, Any]) -> None: return try: - _get_var_info(project, mip, short_name) + _get_var_info( + project, + mip, + short_name, + branding_suffix=branding_suffix, + ) except ValueError as exc: if strict: msg = ( diff --git a/esmvalcore/cmor/table.py b/esmvalcore/cmor/table.py index 7a083ff58d..1955562c30 100644 --- a/esmvalcore/cmor/table.py +++ b/esmvalcore/cmor/table.py @@ -188,7 +188,7 @@ def get_var_info( ) -def load_cmor_tables(cfg: Config) -> None: +def _load_cmor_tables(cfg: Config) -> None: """Load the configured CMOR tables into :data:`esmvalcore.cmor.table.CMOR_TABLES`. Parameters @@ -457,6 +457,9 @@ def __init__( self.tables: dict[str, TableInfo] = {} """A mapping from table names to :class:`TableInfo` objects.""" + def __repr__(self) -> str: + return f"{self.__class__.__name__}(paths={list(self.paths)}, strict={self.strict}, alt_names={self.alt_names})" + def get_table(self, table: str) -> TableInfo | None: """Search and return the table info. @@ -532,6 +535,15 @@ def get_variable( # cmor_strict=False or derived=True var_info = self._look_in_all_tables(derived, alt_names_list) + # If that didn't work either, look in default table if + # cmor_strict=False or derived=True + if not var_info: + var_info = self._look_in_default( + derived, + alt_names_list, + table_name, + ) + # If necessary, adapt frequency of variable (set it to the one from the # requested MIP). E.g., if the user asked for table `Amon`, but the # variable has been found in `day`, use frequency `mon`. @@ -541,6 +553,17 @@ def get_variable( return var_info + def _look_in_default(self, derived, alt_names_list, table_name): + """Look for variable in default table.""" + # TODO: remove in v2.16.0 + var_info = None + if not self.strict or derived: + for alt_names in alt_names_list: + var_info = self.default.get_variable(table_name, alt_names) + if var_info: + break + return var_info + def _look_in_all_tables(self, derived, alt_names_list): """Look for variable in all tables.""" var_info = None @@ -712,7 +735,7 @@ def _load_table(self, json_file): self.var_to_freq[table.name] = {} for var_name, var_data in raw_data["variable_entry"].items(): - var = VariableInfo("CMIP6", name=var_name) + var = VariableInfo("CMIP6") var.read_json(var_data, table.frequency) self._assign_dimensions(var, generic_levels) table[var_name] = var @@ -933,7 +956,6 @@ def __init__( self, table_type: str = "", short_name: str = "", - name: str = "", ) -> None: """Class to read and store variable information. @@ -953,12 +975,8 @@ def __init__( The ``short_name`` parameter is deprecated and will be removed in ESMValCore v2.16.0. - name: - Name of the variable entry in the CMOR table. """ super().__init__() - self.name = name - """Name of the variable entry in the CMOR table.""" self.table_type = table_type self.modeling_realm: list[str] = [] """Modeling realm""" @@ -991,7 +1009,7 @@ def __init__( self._json_data = None def __repr__(self) -> str: - return f"{self.__class__.__name__}(name={self.name})" + return f"<{self.__class__.__name__} defining variable '{self.short_name}'>" def copy(self) -> Self: """Return a shallow copy of VariableInfo. @@ -1305,7 +1323,7 @@ def _read_coordinate(self, value): return coord def _read_variable(self, entry_name, frequency): - var = VariableInfo(table_type="CMIP5", name=entry_name) + var = VariableInfo(table_type="CMIP5") var.frequency = frequency while self._read_line(): key, value = self._last_line_read @@ -1319,7 +1337,8 @@ def _read_variable(self, entry_name, frequency): var.short_name = value if not var.short_name: # Some of our custom CMIP5 table entries are missing the `out_name` field. - var.short_name = var.name + # In that case, we assume the entry name is the same as short_name. + var.short_name = entry_name for dim in var.dimensions: var.coordinates[dim] = self.coords[dim] return var @@ -1454,6 +1473,9 @@ def __init__(self, cmor_tables_path: str | Path | None = None) -> None: for path in self.paths: self._read_table_dir(str(path)) + def __repr__(self) -> str: + return f"{self.__class__.__name__}(paths={list(self.paths)})" + def _read_table_dir(self, table_dir: str) -> None: """Read CMOR tables from directory.""" # If present, read coordinates @@ -1539,6 +1561,9 @@ class NoInfo(InfoBase): def __init__(self) -> None: pass + def __repr__(self) -> str: + return f"{self.__class__.__name__}()" + def get_variable( self, table_name: str, # noqa: ARG002 @@ -1567,7 +1592,7 @@ def get_variable( otherwise. """ - vardef = VariableInfo(name=short_name) + vardef = VariableInfo() vardef.short_name = short_name return vardef diff --git a/esmvalcore/config/_config_validators.py b/esmvalcore/config/_config_validators.py index 6324b374d9..fa9f006f51 100644 --- a/esmvalcore/config/_config_validators.py +++ b/esmvalcore/config/_config_validators.py @@ -14,7 +14,7 @@ from esmvalcore import __version__ as current_version from esmvalcore.cmor.check import CheckLevels -from esmvalcore.cmor.table import load_cmor_tables +from esmvalcore.cmor.table import _load_cmor_tables from esmvalcore.config._config import TASKSEP, load_config_developer from esmvalcore.exceptions import ( ESMValCoreDeprecationWarning, @@ -387,7 +387,7 @@ def validate_cmor_tables(value: dict) -> None: # preferably be avoided. This would require passing around session objects # instead of relying on global state (e.g. esmvalcore.config.CFG, # esmvalcore.cmor.tables.CMOR_TABLES). - load_cmor_tables({"projects": value}) # type: ignore[arg-type] + _load_cmor_tables({"projects": value}) # type: ignore[arg-type] def validate_projects( diff --git a/esmvalcore/preprocessor/_other.py b/esmvalcore/preprocessor/_other.py index 972f93acbb..cc48022804 100644 --- a/esmvalcore/preprocessor/_other.py +++ b/esmvalcore/preprocessor/_other.py @@ -43,6 +43,7 @@ def align_metadata( target_project: str, target_mip: str, target_short_name: str, + target_branding_suffix: str | None = None, strict: bool = True, ) -> Cube: """Set cube metadata to entries from a specific target project. @@ -64,6 +65,8 @@ def align_metadata( MIP table from which target metadata is read. target_short_name: Variable short name from which target metadata is read. + target_branding_suffix: + Branding suffix from which target metadata is read. strict: If ``True``, raise an error if desired metadata cannot be read for variable ``target_short_name`` of MIP table ``target_mip`` and project @@ -86,7 +89,12 @@ def align_metadata( cube = cube.copy() try: - var_info = _get_var_info(target_project, target_mip, target_short_name) + var_info = _get_var_info( + target_project, + target_mip, + target_short_name, + branding_suffix=target_branding_suffix, + ) except ValueError as exc: if strict: raise @@ -101,13 +109,28 @@ def align_metadata( return cube -def _get_var_info(project: str, mip: str, short_name: str) -> VariableInfo: +def _get_var_info( + project: str, + mip: str, + short_name: str, + branding_suffix: str | None, +) -> VariableInfo: """Get variable information.""" - var_info = get_var_info(project, mip, short_name) + var_info = get_var_info( + project, + mip, + short_name, + branding_suffix=branding_suffix, + ) if var_info is None: msg = ( - f"Variable '{short_name}' not available for table '{mip}' of " - f"project '{project}'" + f"Variable '{short_name}' " + + ( + f"with branding suffix '{branding_suffix}' " + if branding_suffix + else "" + ) + + f"not available for table '{mip}' of project '{project}'" ) raise ValueError(msg) return var_info From ed35a661a7b40a40a6b81c42b259423862f628a1 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Wed, 28 Jan 2026 17:21:06 +0100 Subject: [PATCH 08/47] Do not load CMOR tables from built-in config-developer.yml prior to loading them from config --- esmvalcore/cmor/table.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/esmvalcore/cmor/table.py b/esmvalcore/cmor/table.py index 1955562c30..e2e7b5c8e1 100644 --- a/esmvalcore/cmor/table.py +++ b/esmvalcore/cmor/table.py @@ -537,7 +537,7 @@ def get_variable( # If that didn't work either, look in default table if # cmor_strict=False or derived=True - if not var_info: + if not var_info and self.default is not None: var_info = self._look_in_default( derived, alt_names_list, @@ -1595,7 +1595,3 @@ def get_variable( vardef = VariableInfo() vardef.short_name = short_name return vardef - - -# Load the default tables on initializing the module. -read_cmor_tables() From 4e11d10eb4d3f422b2ac6e565ae1c21364fcf2bc Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Wed, 28 Jan 2026 17:26:02 +0100 Subject: [PATCH 09/47] Add note to docs about what to do if CMOR_TABLES is empty --- esmvalcore/cmor/table.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/esmvalcore/cmor/table.py b/esmvalcore/cmor/table.py index e2e7b5c8e1..1e6f65a5e2 100644 --- a/esmvalcore/cmor/table.py +++ b/esmvalcore/cmor/table.py @@ -31,7 +31,12 @@ logger = logging.getLogger(__name__) CMOR_TABLES: dict[str, InfoBase] = {} -"""dict of str, obj: CMOR info objects.""" +"""dict of str, obj: CMOR info objects. + +.. note:: + If this dictionary is empty, it can be populated by loading the global + configuration by importing the :mod:`esmvalcore.config` module. +""" _CMOR_KEYS = ( "standard_name", From ab823bdf23cae56177fd2dfddf60c9047423d070 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Thu, 29 Jan 2026 12:01:07 +0100 Subject: [PATCH 10/47] Add upgrade instructions for users moving away from config-developer.yml --- doc/quickstart/configure.rst | 284 ++++++++++++++++++----------------- 1 file changed, 144 insertions(+), 140 deletions(-) diff --git a/doc/quickstart/configure.rst b/doc/quickstart/configure.rst index 4e39b94d94..a2bcd46931 100644 --- a/doc/quickstart/configure.rst +++ b/doc/quickstart/configure.rst @@ -693,6 +693,10 @@ The following project-specific options are available: - Description - Type - Default value + * - ``cmor_table`` + - :ref:`CMOR tables ` are used to define the variables that ESMValCore can work with. Refer to :ref:`config-cmor-tables` for available options. + - :obj:`dict` + - ``{}`` * - ``data`` - Data sources are used to find input data and have to be configured before running the tool. Refer to :ref:`config-data-sources` for details. - :obj:`dict` @@ -706,6 +710,23 @@ The following project-specific options are available: - :obj:`str` - Refer to :ref:`config-preprocessor-filename-template`. +.. _config-cmor-tables: + +CMOR table configuration +------------------------ + +:ref:`CMOR tables ` are used to define the variables that ESMValCore +can work with. + +Default values are provided in ``defaults/cmor_tables.yml``, +for example: + +.. literalinclude:: ../configurations/defaults/cmor_tables.yml + :language: yaml + :caption: First few lines of ``defaults/cmor_tables.yml`` + :end-before: CMIP3: + + .. _config-data-sources: Data sources @@ -1098,179 +1119,162 @@ resort. Developer configuration file ============================ -Most users and diagnostic developers will not need to change this file, -but it may be useful to understand its content. -The settings from this file are being moved to the -:ref:`new configuration system `. In particular, the -``input_dir``, ``input_file``, and ``ignore_warnings`` settings have already -been replaced by the :class:`esmvalcore.io.local.LocalDataSource` that can be +.. deprecated:: 2.14.0 + + The developer configuration file is deprecated and will no longer be supported + in v2.16.0. Please use the :ref:`project-specific configuration ` + instead. + + See the `v2.13.0 `__ + documentation for previous usage of the developer configuration file. + +.. warning:: + + Make sure that **no** ``config-developer.yml`` file is saved + in the ESMValCore configuration directories (see + :ref:`config_overview` for details), as it does not contain configuration + options that are valid in the new configuration system. + +Upgrade instructions for finding files +-------------------------------------- + +The ``input_dir``, ``input_file``, and ``ignore_warnings`` settings have +been replaced by the :class:`esmvalcore.io.local.LocalDataSource`, which can be configured via :ref:`data sources `. -The developer configuration file will be installed along with ESMValCore and can -also be viewed on GitHub: -`esmvalcore/config-developer.yml -`_. -This configuration file describes the CMOR tables for several -key projects (CMIP6, CMIP5, obs4MIPs, OBS6, OBS), and for native output data for some -models (ICON, IPSL, ... see :ref:`configure_native_models`). -Users can get a copy of this file with default values by running +Example 1: A config-developer.yml file specifying a directory structure for +CMIP6 data: -.. code-block:: bash +.. code:: yaml - esmvaltool config get_config_developer --path=${TARGET_FOLDER} + CMIP6: + input_dir: + ESGF: "{project}/{activity}/{institute}/{dataset}/{exp}/{ensemble}/{mip}/{short_name}/{grid}/{version}" + input_file: "{short_name}_{mip}_{dataset}_{exp}_{ensemble}_{grid}*.nc" -If the option ``--path`` is omitted, the file will be created in -``~/.esmvaltool``. +and associated ``rootpath`` and ``drs`` settings: -.. note:: +.. code:: yaml - Remember to change the configuration option ``config_developer_file`` if you - want to use a custom config developer file. + rootpath: + CMIP6: ~/climate_data + drs: + CMIP6: ESGF -.. warning:: +would translate to the following new configuration: - For now, make sure that the custom ``config-developer.yml`` is **not** saved - in the ESMValTool/Core configuration directories (see - :ref:`config_yaml_files` for details). - This will change in the future due to the :ref:`redesign of ESMValTool/Core's - configuration `. +.. code:: yaml -Example of the CMIP6 project configuration: + projects: + CMIP6: + data: + local: + type: esmvalcore.io.local.LocalDataSource + rootpath: ~/climate_data + dirname_template: "{project}/{activity}/{institute}/{dataset}/{exp}/{ensemble}/{mip}/{short_name}/{grid}/{version}" + filename_template: "{short_name}_{mip}_{dataset}_{exp}_{ensemble}_{grid}*.nc" -.. code-block:: yaml +Upgrade instructions for naming preprocessor output files +--------------------------------------------------------- - CMIP6: - cmor_type: 'CMIP6' - cmor_strict: true +The ``output_file`` setting has been replaced by the +``preprocessor_filename_template`` settings described in +:ref:`config-preprocessor-filename-template`. -.. _cmor_table_configuration: +Example 1: A config-developer.yml file specifying preprocessor output filenames +for CMIP6 data: -Project CMOR table configuration --------------------------------- +.. code:: yaml -ESMValCore comes bundled with several :ref:`CMOR tables `, which are stored in the directory -`esmvalcore/cmor/tables `_. -These are copies of the tables available from `PCMDI `_. + CMIP6: + output_file: "{project}_{dataset}_{mip}_{exp}_{ensemble}_{short_name}_{grid}" -For every ``project`` that can be used in the recipe, there are four settings -related to CMOR table settings available: -* ``cmor_type``: can be ``CMIP5`` if the CMOR table is in the same format as the - CMIP5 table or ``CMIP6`` if the table is in the same format as the CMIP6 table. -* ``cmor_strict``: if this is set to ``false``, the CMOR table will be - extended with variables from the :ref:`custom_cmor_tables` (by default loaded - from the ``esmvalcore/cmor/tables/custom`` directory) and it is possible to - use variables with a ``mip`` which is different from the MIP table in which - they are defined. Note that this option is always enabled for - :ref:`derived variables `. -* ``cmor_path``: path to the CMOR table. - Relative paths are with respect to `esmvalcore/cmor/tables`_. - Defaults to the value provided in ``cmor_type`` written in lower case. -* ``cmor_default_table_prefix``: Prefix that needs to be added to the ``mip`` - to get the name of the file containing the ``mip`` table. - Defaults to the value provided in ``cmor_type``. +would translate to the following new configuration: -.. _custom_cmor_tables: +.. code:: yaml -Custom CMOR tables ------------------- + projects: + CMIP6: + preprocessor_filename_template: "{project}_{dataset}_{mip}_{exp}_{ensemble}_{short_name}_{grid}" -As mentioned in the previous section, the CMOR tables of projects that use -``cmor_strict: false`` will be extended with custom CMOR tables. -For :ref:`derived variables ` (the ones with ``derive: -true`` in the recipe), the custom CMOR tables will always be considered. -By default, these custom tables are loaded from `esmvalcore/cmor/tables/custom -`_. -However, by using the special project ``custom`` in the -``config-developer.yml`` file with the option ``cmor_path``, a custom location -for these custom CMOR tables can be specified. -In this case, the default custom tables are extended with those entries from -the custom location (in case of duplication, the custom location tables take -precedence). +Upgrade instructions for using custom CMOR tables +------------------------------------------------- -Example: +The CMOR tables can now be configured via :ref:`config-cmor-tables`. The +following mapping applies: + +- ``cmor_type`` has been replaced by ``type`` +- ``cmor_strict`` has been replaced by ``strict`` +- ``cmor_path`` has been replaced by ``paths`` +- ``cmor_default_table_prefix`` is no longer needed. + +Because it is now possible to configure multiple paths to directories containing +CMOR tables per project, the ``custom`` project for specifying additional custom +CMOR tables is no longer needed or supported. + +Example 1: A config-developer.yml file specifying different CMIP6 CMOR tables +than the default ones, augmented by the default custom CMOR tables: .. code-block:: yaml - custom: - cmor_path: ~/my/own/custom_tables - -This path can be given as relative path (relative to `esmvalcore/cmor/tables`_) -or as absolute path. -Other options given for this special table will be ignored. - -Custom tables in this directory need to follow the naming convention -``CMOR_{short_name}.dat`` and need to be given in CMIP5 format. - -Example for the file ``CMOR_asr.dat``: - -.. code-block:: - - SOURCE: CMIP5 - !============ - variable_entry: asr - !============ - modeling_realm: atmos - !---------------------------------- - ! Variable attributes: - !---------------------------------- - standard_name: - units: W m-2 - cell_methods: time: mean - cell_measures: area: areacella - long_name: Absorbed shortwave radiation - !---------------------------------- - ! Additional variable information: - !---------------------------------- - dimensions: longitude latitude time - type: real - positive: down - !---------------------------------- - ! - -It is also possible to use a special coordinates file ``CMOR_coordinates.dat``, -which will extend the entries from the default one -(`esmvalcore/cmor/tables/custom/CMOR_coordinates.dat -`_). - - -.. _configure_native_models: - -Configuring datasets in native format -------------------------------------- - -ESMValCore can be configured for handling native model output formats and -specific reanalysis/observation datasets without preliminary reformatting. -These datasets can be either hosted under the ``native6`` project (mostly -native reanalysis/observational datasets) or under a dedicated project, e.g., -``ICON`` (mostly native models). + CMIP6: + cmor_path: /path/to/cmip6-cmor-tables + cmor_strict: true + cmor_type: CMIP6 -Example: +would translate to the following new configuration: .. code-block:: yaml - native6: - cmor_strict: false - cmor_type: 'CMIP6' - cmor_default_table_prefix: 'CMIP6_' + projects: + CMIP6: + cmor_table: + type: esmvalcore.cmor.table.CMIP6Info + strict: true + paths: + - /path/to/cmip6-cmor-tables + - cmip6-custom - ICON: - cmor_strict: false - cmor_type: 'CMIP6' - cmor_default_table_prefix: 'CMIP6_' +where the ``cmip6-custom`` relative path refers to +`esmvalcore/cmor/tables/cmip6-custom `__ +and ``type`` refers to :class:`esmvalcore.cmor.table.CMIP6Info`. -A detailed description on how to add support for further native datasets is -given :ref:`here `. +Example 2: A config-developer.yml file specifying additional custom CMOR tables: -.. hint:: +.. code:: yaml - When using native datasets, it might be helpful to specify a custom location - for the :ref:`custom_cmor_tables`. - This allows reading arbitrary variables from native datasets. - Note that this requires the option ``cmor_strict: false`` in the - :ref:`project configuration ` used for the native - model output. + CMIP6: + cmor_path: cmip6 + cmor_strict: true + cmor_type: CMIP6 + custom: + cmor_path: /path/to/custom-cmip5-style-tables +would translate to the following new configuration: + +.. code:: yaml + + projects: + CMIP6: + cmor_table: + type: esmvalcore.cmor.table.CMIP6Info + strict: true + paths: + - cmip6 + - cmip6-custom + - /path/to/custom-cmip6-style-tables + +where the relative paths ``cmip6`` and ``cmip6-custom`` refer to +`esmvalcore/cmor/tables/cmip6 `__ +and +`esmvalcore/cmor/tables/cmip6-custom `__ +respectively and the directory ``/path/to/custom-cmip6-style-tables`` contains +the additional custom CMOR tables in CMIP6 format. A script to translate custom +CMIP5 format tables to CMIP6 format tables is available +`here `__. +Note that it is no longer possible to mix CMIP6-style tables with custom +CMIP5-style tables for the same project. .. _config-ref: From d08827fff467dc329721c150ec3bce02ec8c3b36 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Thu, 29 Jan 2026 14:44:43 +0100 Subject: [PATCH 11/47] Fix no CMOR table case and update more docs --- doc/develop/fixing_data.rst | 201 ++++++++++++--------- doc/quickstart/configure.rst | 30 ++- esmvalcore/cmor/table.py | 92 ++++++---- esmvalcore/config/_config_validators.py | 4 + esmvalcore/dataset.py | 66 ++++--- tests/unit/config/test_config_validator.py | 20 +- 6 files changed, 258 insertions(+), 155 deletions(-) diff --git a/doc/develop/fixing_data.rst b/doc/develop/fixing_data.rst index 119fef5f09..639e00bd2f 100644 --- a/doc/develop/fixing_data.rst +++ b/doc/develop/fixing_data.rst @@ -316,6 +316,15 @@ missing coordinate you can create a fix for this model: Customizing checker strictness ============================== +The baseline case for ESMValCore input data is fully +:ref:`CMOR compliant ` data and this is checked when data is +loaded by :meth:`esmvalcore.dataset.Dataset.load`. + +However, it is possible to disable these checks completely by configuring +the project so it uses the :class:`esmvalcore.cmor.table.NoInfo` CMOR table, +or to adjust the strictness of the checks using the :ref:`configuration option +` ``check_level``. + The data checker classifies its issues using four different levels of severity. From highest to lowest: @@ -345,6 +354,13 @@ below from the lowest level of strictness to the highest: strictness. Mostly useful for checking datasets that you have produced, to be sure that future users will not be distracted by inoffensive warnings. +.. warning:: + + While it is possible to work with datasets that are not described in a CMOR + table or only partially follow the CMOR standards, the + :ref:`preprocessor functions ` and + :ref:`diagnostics ` have been designed to work with + CMORized data and may not work as expected with non-CMORized data. .. _add_new_fix_native_datasets: @@ -357,14 +373,23 @@ under project ``native6``. .. _add_new_fix_native_datasets_config: -Configuration -------------- +CMOR Table Configuration +------------------------ + +An example of a CMOR table configuration for projects used for native datasets is given in + +.. literalinclude:: ../configurations/defaults/cmor_tables.yml + :language: yaml + :caption: Example native data format projects in ``defaults/cmor_tables.yml`` + :prepend: projects: + :start-at: # Observational and reanalysis data that can be read in its native format by ESMValCore. + :end-before: CESM: -An example of a configuration in ``config-developer.yml`` for projects used for -native datasets is given :ref:`here `. -Make sure to use the option ``cmor_strict: false`` for these projects if you -want to make use of :ref:`custom_cmor_tables`. -This allows reading arbitrary variables from native datasets. +The option ``strict: false`` is convenient for these projects, if you +want to make use of the feature that looks in all tables instead of +only the one specified by the ``mip`` facet in the recipe or +:class:`~esmvalcore.dataset.Dataset`: like this, a custom variable only needs to +be defined for a single ``mip`` and can then be used with all ``mip`` s. .. _add_new_fix_native_datasets_locate_data: @@ -373,89 +398,85 @@ Locate data To allow ESMValCore to locate the data files, use the following steps: - - If you want to use the ``native6`` project (recommended for datasets whose - input files can be easily moved to the usual ``native6`` directory - structure given by the :ref:`configuration option ` - ``rootpath``; this is usually the case for native reanalysis/observational - datasets): - - The entry ``native6`` of ``config-developer.yml`` should be complemented - with sub-entries for ``input_dir`` and ``input_file`` that go under a new - key representing the data organization (such as ``MY_DATA_ORG``), and - these sub-entries can use an arbitrary list of ``{placeholders}``. - Example : - - .. code-block:: yaml - - native6: - ... - input_dir: - default: 'Tier{tier}/{dataset}/{version}/{frequency}/{short_name}' - MY_DATA_ORG: '{dataset}/{exp}/{simulation}/{version}/{type}' - input_file: - default: '*.nc' - MY_DATA_ORG: '{simulation}_*.nc' - ... - - To find your native data (e.g., called ``MYDATA``) that is for example - located in ``{rootpath}/MYDATA/amip/run1/42-0/atm/run1_1979.nc`` - (``{rootpath}`` is ESMValTool's ``rootpath`` :ref:`configuration option - ` for the project ``native6``), use the following dataset - entry in your recipe - - .. code-block:: yaml - - datasets: - - {project: native6, dataset: MYDATA, exp: amip, simulation: run1, version: 42-0, type: atm} - - and make sure to use the following :ref:`configuration option - ` ``drs``: - - .. code-block:: yaml - - drs: - native6: MY_DATA_ORG - - - If you want to use a dedicated project for your native dataset - (recommended for datasets for which you cannot control the location of the - input files; this is usually the case for native model output): - - A new entry for the project needs to be added to ``config-developer.yml``. - For example, for the ICON model, create a new project ``ICON``: - - .. code-block:: yaml - - ICON: - ... - input_dir: - default: - - '{exp}' - - '{exp}/outdata' - - '{exp}/output' - input_file: - default: '{exp}_{var_type}*.nc' - ... - - To find your ICON data that is for example located in files like - ``{rootpath}/amip/amip_atm_2d_ml_20000101T000000Z.nc`` (``{rootpath}`` is - ESMValCore's :ref:`configuration option ` ``rootpath`` for - the project ``ICON``), use the following dataset entry in your recipe: - - .. code-block:: yaml - - datasets: - - {project: ICON, dataset: ICON, exp: amip} - - Please note the duplication of the name ``ICON`` in ``project`` and - ``dataset``, which is necessary to comply with ESMValTool's data finding - and CMORizing functionalities. - For other native models, ``dataset`` could also refer to a subversion of - the model. - Note that it is possible to predefine facets via :ref:`extra facets - `. - In this ICON example, the facet ``var_type`` is :download:`predefined - ` - for many variables. +- If you want to use the ``native6`` project, recommended for datasets whose + input files can be easily moved to the usual ``native6`` directory + structure given by + + .. literalinclude:: ../configurations/data-local-esmvaltool.yml + :language: yaml + :caption: ``native6`` standard directory organization in ``data-local-esmvaltool.yml`` + :end-before: # Data that has been CMORized by ESMValTool according to the CMIP6 standard. + + this is preferred. This is usually the case for native reanalysis/observational + datasets. + +- If moving the data into a particular directory structure is not possible, + the ``data`` entry of the ``native6`` project could be complemented + with another data source that goes under a new + key representing the data organization (such as ``MY_DATA_ORG``), and + these sub-entries can use an arbitrary list of ``{placeholders}``. + + Example: + + .. code-block:: yaml + + projects: + native6: + data: + MY_DATA_ORG: + type: esmvalcore.io.local.LocalDataSource + rootpath: /path/to/data + dirname_template: "{dataset}/{exp}/{simulation}/{version}/{type}" + filename_template: '{simulation}_*.nc' + + would allow the tool to find your native data (e.g., a ``dataset`` called ``MYDATA``) + that is for example located in ``/path/to/data/MYDATA/amip/run1/42-0/atm/run1_1979.nc`` + if you use the following dataset entry in your recipe + + .. code-block:: yaml + + datasets: + - {project: native6, dataset: MYDATA, exp: amip, simulation: run1, version: 42-0, type: atm} + +- If you want to use a dedicated project for your native dataset + (this is usually the case for native model output): + + A new entry for the project needs to be added under :ref:`config-projects`. + For example, for the ICON model, create a new project ``ICON`` and define + its data sources: + + .. literalinclude:: ../configurations/data-native-icon.yml + :language: yaml + :caption: ``ICON`` standard directory organization in ``data-native-icon.yml`` + + and a CMOR table configuration: + + .. literalinclude:: ../configurations/defaults/cmor_tables.yml + :language: yaml + :caption: ``ICON`` CMOR table configuration from ``defaults/cmor_tables.yml`` + :prepend: projects: + :start-at: ICON: + :end-at: strict: false + + To find your ICON data that is for example located in files like + ``~/climate_data/amip/amip_atm_2d_ml_20000101T000000Z.nc``, use the following + dataset entry in your recipe: + + .. code-block:: yaml + + datasets: + - {project: ICON, dataset: ICON, exp: amip} + + Please note the duplication of the name ``ICON`` in ``project`` and + ``dataset``, which is necessary to comply with ESMValTool's data finding + and CMORizing functionalities. + For other native models, ``dataset`` could also refer to a subversion of + the model. + Note that it is possible to predefine facets via :ref:`extra facets + `. + In this ICON example, the facet ``var_type`` is :download:`predefined + ` + for many variables. .. _add_new_fix_native_datasets_fix_data: diff --git a/doc/quickstart/configure.rst b/doc/quickstart/configure.rst index a2bcd46931..a973754269 100644 --- a/doc/quickstart/configure.rst +++ b/doc/quickstart/configure.rst @@ -694,7 +694,7 @@ The following project-specific options are available: - Type - Default value * - ``cmor_table`` - - :ref:`CMOR tables ` are used to define the variables that ESMValCore can work with. Refer to :ref:`config-cmor-tables` for available options. + - :ref:`CMOR tables ` are used to define the variables that ESMValCore can work with. Refer to :ref:`cmor_table_configuration` for available options. - :obj:`dict` - ``{}`` * - ``data`` @@ -710,7 +710,7 @@ The following project-specific options are available: - :obj:`str` - Refer to :ref:`config-preprocessor-filename-template`. -.. _config-cmor-tables: +.. _cmor_table_configuration: CMOR table configuration ------------------------ @@ -726,6 +726,30 @@ for example: :caption: First few lines of ``defaults/cmor_tables.yml`` :end-before: CMIP3: +The ``type`` parameter defines which class is used to read the CMOR tables for a +project, it should be a subclass of :class:`esmvalcore.cmor.table.InfoBase`. +The other parameters are passed as keyword arguments to the class when it is +created. See :mod:`esmvalcore.cmor.table` for a description of the built-in +classes for reading CMOR tables and their parameters. + +Most users will not need to change the CMOR table configuration. However, if you +have variables you would like to use which are not in the standard tables, +it is recommended that you start from the default configuration and extend the +list of ``paths`` with the directory where your custom CMOR tables are stored. + +If you have data that is not described in a CMOR table at all, you can use +the :class:`esmvalcore.cmor.table.NoInfo` class to indicate that no CMOR table +is available. In that case you will need to provide all necessary facets +for finding and saving the data in the :ref:`recipe ` or +:class:`~esmvalcore.dataset.Dataset`, and +:ref:`CMOR checks ` will be skipped. + +.. warning:: + + While it is possible to work with datasets that are not described in a CMOR + table,the :ref:`preprocessor functions ` and + :ref:`diagnostics ` have been designed to work with + CMORized data and may not work as expected with non-CMORized data. .. _config-data-sources: @@ -1201,7 +1225,7 @@ would translate to the following new configuration: Upgrade instructions for using custom CMOR tables ------------------------------------------------- -The CMOR tables can now be configured via :ref:`config-cmor-tables`. The +The CMOR tables can now be configured via :ref:`cmor_table_configuration`. The following mapping applies: - ``cmor_type`` has been replaced by ``type`` diff --git a/esmvalcore/cmor/table.py b/esmvalcore/cmor/table.py index 1e6f65a5e2..6ad40b174a 100644 --- a/esmvalcore/cmor/table.py +++ b/esmvalcore/cmor/table.py @@ -374,9 +374,9 @@ def get_tables( tables = cls(**kwargs) if not isinstance(tables, InfoBase): msg = ( - "Expected CMOR tables of type `esmvalcore.cmor.table.InfoBase`, " + "`type` should be a subclass `esmvalcore.cmor.table.InfoBase`, " f"but your configuration for project '{project}' contains " - f"'{tables}' of type '{type(tables)}'." + f"'{tables}' of type: '{type(tables)}'." ) raise TypeError(msg) _TABLE_CACHE[cache_key] = tables @@ -404,15 +404,17 @@ class InfoBase: `variable_alt_names.yml `_ will be used. - strict: bool - If False, will look for a variable in other tables if it can not be - found in the requested one. + strict: + If :obj:`False`, the function :meth:`~esmvalcore.cmor.table.InfoBase.get_variable` + will look for a variable in other tables if it can not be found in the + table specified by ``mip`` in the :ref:`recipe ` or :class:`~esmvalcore.dataset.Dataset`. paths: - A list of paths to CMOR tables. If the path is relative and exists in - the installed copy of the + A list of paths to CMOR tables. The path can be relative to the built-in + tables in the `esmvalcore/cmor/tables `_ - directory it will be used. + directory, or any other path. The built-in tables will be used if the + path is relative and exists in the built-in tables directory. """ def __init__( @@ -494,7 +496,8 @@ def get_variable( Parameters ---------- table_name: - Table name, i.e., the variable's MIP. + Table name, i.e., the ``mip`` in the :ref:`recipe ` or + :class:`~esmvalcore.dataset.Dataset`. short_name: Variable's short name. branding_suffix: @@ -638,9 +641,11 @@ class CMIP6Info(InfoBase): the default values from the installed copy of `variable_alt_names.yml`_ will be used. - strict: bool - If False, will look for a variable in other tables if it can not be - found in the requested one. + strict: + If :obj:`False`, the function :meth:`~esmvalcore.cmor.table.InfoBase.get_variable` + will look for a variable in other tables if it can not be found in the + table specified by ``mip`` in the :ref:`recipe ` or + :class:`~esmvalcore.dataset.Dataset`. default_table_prefix: If the table_id contains a prefix, it can be specified here. @@ -651,9 +656,12 @@ class CMIP6Info(InfoBase): ESMValCore v2.16.0. paths: - A list of paths to CMOR tables. If the path is relative and exists in - the installed copy of the `esmvalcore/cmor/tables`_ directory it will - be used. + A list of paths to CMOR tables. The path can be relative to the built-in + tables in the + `esmvalcore/cmor/tables `_ + directory, or any other path. The built-in tables will be used if the + path is relative and exists in the built-in tables directory. + """ def __init__( @@ -842,14 +850,18 @@ class Obs4MIPsInfo(CMIP6Info): the default values from the installed copy of `variable_alt_names.yml`_ will be used. - strict: bool - If False, will look for a variable in other tables if it can not be - found in the requested one. + strict: + If :obj:`False`, the function :meth:`~esmvalcore.cmor.table.InfoBase.get_variable` + will look for a variable in other tables if it can not be found in the + table specified by ``mip`` in the :ref:`recipe ` or + :class:`~esmvalcore.dataset.Dataset`. paths: - A list of paths to CMOR tables. If the path is relative and exists in - the ``tables`` directory in :mod:`esmvalcore.cmor`, the version of the - tables shipped with ESMValCore will be used. + A list of paths to CMOR tables. The path can be relative to the built-in + tables in the + `esmvalcore/cmor/tables `_ + directory, or any other path. The built-in tables will be used if the + path is relative and exists in the built-in tables directory. """ def __init__( @@ -1194,9 +1206,11 @@ class CMIP5Info(InfoBase): the default values from the installed copy of `variable_alt_names.yml`_ will be used. - strict: bool - If False, will look for a variable in other tables if it can not be - found in the requested one. + strict: + If :obj:`False`, the function :meth:`~esmvalcore.cmor.table.InfoBase.get_variable` + will look for a variable in other tables if it can not be found in the + table specified by ``mip`` in the :ref:`recipe ` or + :class:`~esmvalcore.dataset.Dataset`. default_table_prefix: If the table_id contains a prefix, it can be specified here. @@ -1207,9 +1221,11 @@ class CMIP5Info(InfoBase): ESMValCore v2.16.0. paths: - A list of paths to CMOR tables. If the path is relative and exists in - the installed copy of the `esmvalcore/cmor/tables`_ directory it will - be used. + A list of paths to CMOR tables. The path can be relative to the built-in + tables in the + `esmvalcore/cmor/tables `_ + directory, or any other path. The built-in tables will be used if the + path is relative and exists in the built-in tables directory. """ @@ -1393,9 +1409,11 @@ class CMIP3Info(CMIP5Info): the default values from the installed copy of `variable_alt_names.yml`_ will be used. - strict: bool - If False, will look for a variable in other tables if it can not be - found in the requested one. + strict: + If :obj:`False`, the function :meth:`~esmvalcore.cmor.table.InfoBase.get_variable` + will look for a variable in other tables if it can not be found in the + table specified by ``mip`` in the :ref:`recipe ` or + :class:`~esmvalcore.dataset.Dataset`. default_table_prefix: If the table_id contains a prefix, it can be specified here. @@ -1406,9 +1424,11 @@ class CMIP3Info(CMIP5Info): ESMValCore v2.16.0. paths: - A list of paths to CMOR tables. If the path is relative and exists in - the installed copy of the `esmvalcore/cmor/tables`_ directory it will - be used. + A list of paths to CMOR tables. The path can be relative to the built-in + tables in the + `esmvalcore/cmor/tables `_ + directory, or any other path. The built-in tables will be used if the + path is relative and exists in the built-in tables directory. """ @@ -1582,9 +1602,13 @@ def get_variable( Parameters ---------- table_name: - Table name, i.e., the variable's MIP. + Table name, i.e., the ``mip`` in the :ref:`recipe ` or + :class:`~esmvalcore.dataset.Dataset`. short_name: Variable's short name. + branding_suffix: + A suffix that will be appended to ``short_name`` when looking up the + variable in the CMOR table. derived: Variable is derived. Information retrieval for derived variables always looks in the default tables (usually, the custom tables) if diff --git a/esmvalcore/config/_config_validators.py b/esmvalcore/config/_config_validators.py index fa9f006f51..7f4e4b5361 100644 --- a/esmvalcore/config/_config_validators.py +++ b/esmvalcore/config/_config_validators.py @@ -402,6 +402,10 @@ def validate_projects( "preprocessor_filename_template": validate_string, } for project, project_config in mapping.items(): + if "cmor_table" not in project_config: + project_config["cmor_table"] = { + "type": "esmvalcore.cmor.table.NoInfo", + } for option, val in project_config.items(): if option not in options_for_project: msg = ( diff --git a/esmvalcore/dataset.py b/esmvalcore/dataset.py index 8a6a20c156..94ee3df9a8 100644 --- a/esmvalcore/dataset.py +++ b/esmvalcore/dataset.py @@ -19,9 +19,11 @@ from esmvalcore._recipe import check from esmvalcore._recipe.from_datasets import datasets_to_recipe from esmvalcore.cmor.table import ( + NoInfo, _get_branding_suffixes, _get_mips, _update_cmor_facets, + get_tables, ) from esmvalcore.config import CFG from esmvalcore.config._config import load_extra_facets @@ -859,28 +861,39 @@ def _load(self) -> Cube: self.session._fixed_file_dir, # noqa: SLF001 self._get_joined_summary_facets("_", join_lists=True) + "_", ) + cmor_tables_available = not isinstance( + get_tables( + session=self.session, + project=self.facets["project"], # type: ignore[arg-type] + ), + NoInfo, + ) settings: dict[str, dict[str, Any]] = {} - settings["fix_file"] = { - "output_dir": fix_dir_prefix, - "add_unique_suffix": True, - "session": self.session, - **self.facets, - } + if cmor_tables_available: + settings["fix_file"] = { + "output_dir": fix_dir_prefix, + "add_unique_suffix": True, + "session": self.session, + **self.facets, + } settings["load"] = {} - settings["fix_metadata"] = { - "session": self.session, - **self.facets, - } + if cmor_tables_available: + settings["fix_metadata"] = { + "session": self.session, + **self.facets, + } settings["concatenate"] = {"check_level": self.session["check_level"]} - settings["cmor_check_metadata"] = { - "check_level": self.session["check_level"], - "cmor_table": self.facets["project"], - "mip": self.facets["mip"], - "frequency": self.facets["frequency"], - "short_name": self.facets["short_name"], - "branding_suffix": self.facets.get("branding_suffix"), - } + + if cmor_tables_available: + settings["cmor_check_metadata"] = { + "check_level": self.session["check_level"], + "cmor_table": self.facets["project"], + "mip": self.facets["mip"], + "frequency": self.facets["frequency"], + "short_name": self.facets["short_name"], + "branding_suffix": self.facets.get("branding_suffix"), + } if "timerange" in self.facets: settings["clip_timerange"] = { "timerange": self.facets["timerange"], @@ -889,14 +902,15 @@ def _load(self) -> Cube: "session": self.session, **self.facets, } - settings["cmor_check_data"] = { - "check_level": self.session["check_level"], - "cmor_table": self.facets["project"], - "mip": self.facets["mip"], - "frequency": self.facets["frequency"], - "short_name": self.facets["short_name"], - "branding_suffix": self.facets.get("branding_suffix"), - } + if cmor_tables_available: + settings["cmor_check_data"] = { + "check_level": self.session["check_level"], + "cmor_table": self.facets["project"], + "mip": self.facets["mip"], + "frequency": self.facets["frequency"], + "short_name": self.facets["short_name"], + "branding_suffix": self.facets.get("branding_suffix"), + } result: Sequence[PreprocessorItem] = self.files for step, kwargs in settings.items(): diff --git a/tests/unit/config/test_config_validator.py b/tests/unit/config/test_config_validator.py index 44e92e86cc..06fbac5936 100644 --- a/tests/unit/config/test_config_validator.py +++ b/tests/unit/config/test_config_validator.py @@ -228,10 +228,26 @@ def generate_validator_testcases(valid): { "validator": validate_projects, "success": ( - ({"CMIP6": {}}, {"CMIP6": {}}), + ( + {"CMIP6": {}}, + { + "CMIP6": { + "cmor_table": { + "type": "esmvalcore.cmor.table.NoInfo", + }, + }, + }, + ), ( {"CMIP6": {"extra_facets": {}}}, - {"CMIP6": {"extra_facets": {}}}, + { + "CMIP6": { + "cmor_table": { + "type": "esmvalcore.cmor.table.NoInfo", + }, + "extra_facets": {}, + }, + }, ), ), "fail": ( From 01129d4486e0d17e7e4b01ba3c4800b1c767da68 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Thu, 29 Jan 2026 15:05:43 +0100 Subject: [PATCH 12/47] Fix documentation build --- doc/changelog.rst | 2 +- doc/quickstart/find_data.rst | 9 --------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/doc/changelog.rst b/doc/changelog.rst index 4a2af7ba42..e6ba93f230 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -813,7 +813,7 @@ Bug fixes ~~~~~~~~~ - Respect ``ignore_warnings`` settings from the project configuration in config-developer.yml in :func:`esmvalcore.dataset.Dataset.load` (:pull:`2046`) by :user:`schlunma` -- Fixed usage of custom location for :ref:`custom CMOR tables ` (:pull:`2052`) by :user:`schlunma` +- Fixed usage of custom location for custom CMOR tables (:pull:`2052`) by :user:`schlunma` - Fix issue with writing index.html when :ref:`running a recipe ` with ``--resume-from`` (:pull:`2055`) by :user:`bouweandela` - Fixed bug in ICON CMORizer that lead to shifted time coordinates (:pull:`2038`) by :user:`schlunma` - Include ``-`` in allowed characters for bibtex references (:pull:`2097`) by :user:`alistairsellar` diff --git a/doc/quickstart/find_data.rst b/doc/quickstart/find_data.rst index fa1a07705e..76cbd61e2e 100644 --- a/doc/quickstart/find_data.rst +++ b/doc/quickstart/find_data.rst @@ -85,15 +85,6 @@ project, e.g., ``ICON`` (mostly native models). A detailed description of how to include new native datasets is given :ref:`here `. -.. hint:: - - When using native datasets, it might be helpful to specify a custom location - for the :ref:`custom_cmor_tables`. - This allows reading arbitrary variables from native datasets. - Note that this requires the option ``cmor_strict: false`` in the - :ref:`project configuration ` used for the native - model output. - .. _read_native_obs: Supported native reanalysis/observational datasets From f80b1a4acb8064e76f798deb6fd5c23cdd6f3e50 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Thu, 29 Jan 2026 15:13:06 +0100 Subject: [PATCH 13/47] Further docs improvements --- doc/develop/fixing_data.rst | 5 +++-- doc/quickstart/configure.rst | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/doc/develop/fixing_data.rst b/doc/develop/fixing_data.rst index 639e00bd2f..350325fb98 100644 --- a/doc/develop/fixing_data.rst +++ b/doc/develop/fixing_data.rst @@ -320,8 +320,9 @@ The baseline case for ESMValCore input data is fully :ref:`CMOR compliant ` data and this is checked when data is loaded by :meth:`esmvalcore.dataset.Dataset.load`. -However, it is possible to disable these checks completely by configuring -the project so it uses the :class:`esmvalcore.cmor.table.NoInfo` CMOR table, +However, it is possible to disable these checks completely by +:ref:`configuring the project ` so it uses the +:class:`esmvalcore.cmor.table.NoInfo` CMOR table, or to adjust the strictness of the checks using the :ref:`configuration option ` ``check_level``. diff --git a/doc/quickstart/configure.rst b/doc/quickstart/configure.rst index a973754269..5e462be3b8 100644 --- a/doc/quickstart/configure.rst +++ b/doc/quickstart/configure.rst @@ -737,10 +737,11 @@ have variables you would like to use which are not in the standard tables, it is recommended that you start from the default configuration and extend the list of ``paths`` with the directory where your custom CMOR tables are stored. -If you have data that is not described in a CMOR table at all, you can use +If you have data that is not described in a CMOR table at all and do not need +interoperability with typical climate model data, you can use the :class:`esmvalcore.cmor.table.NoInfo` class to indicate that no CMOR table is available. In that case you will need to provide all necessary facets -for finding and saving the data in the :ref:`recipe ` or +for loading and saving the data in the :ref:`recipe ` or :class:`~esmvalcore.dataset.Dataset`, and :ref:`CMOR checks ` will be skipped. From d14d09fd74b91207d74a31ce70b9eb6a4ed005a3 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Thu, 29 Jan 2026 15:45:06 +0100 Subject: [PATCH 14/47] Add test --- esmvalcore/config/_config_validators.py | 4 +++- .../integration/cmor/test_read_cmor_tables.py | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/esmvalcore/config/_config_validators.py b/esmvalcore/config/_config_validators.py index 7f4e4b5361..de16d76a13 100644 --- a/esmvalcore/config/_config_validators.py +++ b/esmvalcore/config/_config_validators.py @@ -476,7 +476,9 @@ def _handle_deprecation( f"been deprecated in ESMValCore version {deprecated_version} and is " f"scheduled for removal in version {remove_version}.{more_info}" ) - warnings.warn(deprecation_msg, ESMValCoreDeprecationWarning, stacklevel=2) + # This function is called by the deprecation functions, which are called by + # ValidatedConfig.__setitem__, so the calling site is 4 levels away. + warnings.warn(deprecation_msg, ESMValCoreDeprecationWarning, stacklevel=4) # TODO: remove in v2.15.0 diff --git a/tests/integration/cmor/test_read_cmor_tables.py b/tests/integration/cmor/test_read_cmor_tables.py index efd296219b..11fe20a84a 100644 --- a/tests/integration/cmor/test_read_cmor_tables.py +++ b/tests/integration/cmor/test_read_cmor_tables.py @@ -7,6 +7,8 @@ import pytest import yaml +import esmvalcore +import esmvalcore.cmor.table from esmvalcore.cmor.table import ( CMOR_TABLES, VariableInfo, @@ -64,6 +66,23 @@ def test_read_cmor_tables(): assert table.strict is False +def test_load_cmor_tables_with_config_developer( + monkeypatch: pytest.MonkeyPatch, + session: Session, +) -> None: + """Test that _load_cmor_tables falls back to config-developer.yml.""" + # TODO: remove in v2.16.0 + cmor_tables: dict[str, esmvalcore.cmor.table.InfoBase] = {} + monkeypatch.setattr(esmvalcore.cmor.table, "CMOR_TABLES", cmor_tables) + monkeypatch.delitem(session, "projects") + monkeypatch.setitem( + session, + "config_developer_file", + Path(esmvalcore.__file__).parent / "config-developer.yml", + ) + assert cmor_tables + + @pytest.mark.parametrize( ( "project", From e2dfa5dad003bba9bd50a17bc7d46adce92dcbc1 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Thu, 29 Jan 2026 16:47:00 +0100 Subject: [PATCH 15/47] Improve test coverage --- esmvalcore/cmor/table.py | 15 ------------ esmvalcore/config/_config_validators.py | 13 +++++++--- .../integration/cmor/test_read_cmor_tables.py | 24 +++++-------------- 3 files changed, 16 insertions(+), 36 deletions(-) diff --git a/esmvalcore/cmor/table.py b/esmvalcore/cmor/table.py index 6ad40b174a..5dd58659d2 100644 --- a/esmvalcore/cmor/table.py +++ b/esmvalcore/cmor/table.py @@ -193,21 +193,6 @@ def get_var_info( ) -def _load_cmor_tables(cfg: Config) -> None: - """Load the configured CMOR tables into :data:`esmvalcore.cmor.table.CMOR_TABLES`. - - Parameters - ---------- - cfg: - The configuration. - """ - CMOR_TABLES.clear() - if cfg.get("config_developer_file") is not None: - read_cmor_tables(cfg["config_developer_file"]) - for project in cfg["projects"]: - CMOR_TABLES[project] = get_tables(cfg, project) - - def read_cmor_tables(cfg_developer: Path | None = None) -> None: """Read cmor tables required in the configuration. diff --git a/esmvalcore/config/_config_validators.py b/esmvalcore/config/_config_validators.py index de16d76a13..6cf3a05b83 100644 --- a/esmvalcore/config/_config_validators.py +++ b/esmvalcore/config/_config_validators.py @@ -12,9 +12,9 @@ from packaging import version +import esmvalcore.cmor.table from esmvalcore import __version__ as current_version from esmvalcore.cmor.check import CheckLevels -from esmvalcore.cmor.table import _load_cmor_tables from esmvalcore.config._config import TASKSEP, load_config_developer from esmvalcore.exceptions import ( ESMValCoreDeprecationWarning, @@ -287,7 +287,7 @@ def validate_config_developer(value): path = validate_path_or_none(value) if path is not None: # This has the side-effect of updating `esmvalcore.config._config.CFG` - # `esmvalcore.cmor.tables.CMOR_TABLES`. + # and `esmvalcore.cmor.tables.CMOR_TABLES`. load_config_developer(path) return path @@ -387,7 +387,14 @@ def validate_cmor_tables(value: dict) -> None: # preferably be avoided. This would require passing around session objects # instead of relying on global state (e.g. esmvalcore.config.CFG, # esmvalcore.cmor.tables.CMOR_TABLES). - _load_cmor_tables({"projects": value}) # type: ignore[arg-type] + esmvalcore.cmor.table.CMOR_TABLES.clear() + for project in value: + esmvalcore.cmor.table.CMOR_TABLES[project] = ( + esmvalcore.cmor.table.get_tables( + session={"projects": value}, # type: ignore[arg-type] + project=project, + ) + ) def validate_projects( diff --git a/tests/integration/cmor/test_read_cmor_tables.py b/tests/integration/cmor/test_read_cmor_tables.py index 11fe20a84a..6a4878e745 100644 --- a/tests/integration/cmor/test_read_cmor_tables.py +++ b/tests/integration/cmor/test_read_cmor_tables.py @@ -7,7 +7,6 @@ import pytest import yaml -import esmvalcore import esmvalcore.cmor.table from esmvalcore.cmor.table import ( CMOR_TABLES, @@ -66,23 +65,6 @@ def test_read_cmor_tables(): assert table.strict is False -def test_load_cmor_tables_with_config_developer( - monkeypatch: pytest.MonkeyPatch, - session: Session, -) -> None: - """Test that _load_cmor_tables falls back to config-developer.yml.""" - # TODO: remove in v2.16.0 - cmor_tables: dict[str, esmvalcore.cmor.table.InfoBase] = {} - monkeypatch.setattr(esmvalcore.cmor.table, "CMOR_TABLES", cmor_tables) - monkeypatch.delitem(session, "projects") - monkeypatch.setitem( - session, - "config_developer_file", - Path(esmvalcore.__file__).parent / "config-developer.yml", - ) - assert cmor_tables - - @pytest.mark.parametrize( ( "project", @@ -142,6 +124,12 @@ def test_get_tables( assert vardef.units +def test_clear_table_cache(session: Session) -> None: + assert esmvalcore.cmor.table._TABLE_CACHE + esmvalcore.cmor.table.clear_table_cache() + assert not esmvalcore.cmor.table._TABLE_CACHE + + CMOR_NEWVAR_ENTRY = dedent( """ !============ From 74f27cc90b38ce3d9916026ea47cd72bbfeed137 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Thu, 29 Jan 2026 17:23:00 +0100 Subject: [PATCH 16/47] Test failure to load cmor tables cases --- .../integration/cmor/test_read_cmor_tables.py | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/tests/integration/cmor/test_read_cmor_tables.py b/tests/integration/cmor/test_read_cmor_tables.py index 6a4878e745..efc3e1175c 100644 --- a/tests/integration/cmor/test_read_cmor_tables.py +++ b/tests/integration/cmor/test_read_cmor_tables.py @@ -124,6 +124,52 @@ def test_get_tables( assert vardef.units +def test_get_tables_unknown_project( + session: Session, +) -> None: + with pytest.raises( + ValueError, + match=r"Unknown project 'unknown', please configure it under 'projects'.", + ): + get_tables(session, "unknown") + + +def test_get_tables_no_type( + session: Session, +) -> None: + session["projects"]["test"] = {"cmor_table": {}} + with pytest.raises( + ValueError, + match=( + r"Missing CMOR table 'type' in configuration of project " + r"test. Current configuration is:\n{}\n" + ), + ): + get_tables(session, "test") + + +class InvalidTable: + pass + + +def test_get_tables_invalid_type( + session: Session, +) -> None: + session["projects"]["test"] = { + "cmor_table": { + "type": "tests.integration.cmor.test_read_cmor_tables.InvalidTable", + }, + } + with pytest.raises( + TypeError, + match=( + r"`type` should be a subclass `esmvalcore.cmor.table.InfoBase`, " + r"but your configuration for project 'test' contains " + ), + ): + get_tables(session, "test") + + def test_clear_table_cache(session: Session) -> None: assert esmvalcore.cmor.table._TABLE_CACHE esmvalcore.cmor.table.clear_table_cache() From 0cba683ad4226c441dfd26d8851fe60501fb6b3e Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Thu, 29 Jan 2026 17:40:55 +0100 Subject: [PATCH 17/47] More tests --- tests/integration/cmor/test_table.py | 32 ++++++++++++++++++---------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/tests/integration/cmor/test_table.py b/tests/integration/cmor/test_table.py index 44feca13c5..07cf326fc1 100644 --- a/tests/integration/cmor/test_table.py +++ b/tests/integration/cmor/test_table.py @@ -7,6 +7,7 @@ import pytest import esmvalcore.cmor +import esmvalcore.cmor.table from esmvalcore.cmor.table import ( CMIP3Info, CMIP5Info, @@ -78,21 +79,25 @@ def test_update_cmor_facets_facet_not_in_table(mocker): class TestCMIP6Info(unittest.TestCase): """Test for the CMIP6 info class.""" - @classmethod - def setUpClass(cls): - """Set up tests. - - We read CMIP6Info once to keep tests times manageable - """ - cls.variables_info = CMIP6Info( + def setUp(self): + self.variables_info = CMIP6Info( paths=[ Path("cmip6/Tables"), Path("cmip6-custom"), ], ) - def setUp(self): - self.variables_info.strict = True + def test_repr(self) -> None: + builtin_tables_path = Path(esmvalcore.cmor.__file__).parent / "tables" + expected_paths = [ + builtin_tables_path / "cmip6" / "Tables", + builtin_tables_path / "cmip6-custom", + ] + result = repr(self.variables_info) + assert result.startswith( + f"CMIP6Info(paths={expected_paths}, strict=True, alt_names=", + ) + assert result.endswith(")") def test_custom_tables_location(self): """Test constructor with custom tables location.""" @@ -167,11 +172,16 @@ def test_get_activity_from_exp(self): activity = self.variables_info.activities["1pctCO2"] self.assertListEqual(activity, ["CMIP"]) - def test_invalid_path(self): + def test_invalid_path(self) -> None: path = Path(__file__) / "path" / "does" / "not" / "exist" msg = r"CMOR tables not found in" with pytest.raises(ValueError, match=msg): - CMIP6Info(path) + CMIP6Info(str(path)) + + def test_invalid_paths(self) -> None: + path = Path(__file__) / "path" / "does" / "not" / "exist" + with pytest.raises(NotADirectoryError, match=str(path)): + CMIP6Info(paths=[path]) class Testobs4mipsInfo(unittest.TestCase): From adff349a08667255a70c997b5f3cbb3b44193e2e Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Thu, 29 Jan 2026 21:41:59 +0100 Subject: [PATCH 18/47] Add another test and update to pytest style tests --- tests/integration/cmor/test_table.py | 524 +++++++++++++-------------- 1 file changed, 243 insertions(+), 281 deletions(-) diff --git a/tests/integration/cmor/test_table.py b/tests/integration/cmor/test_table.py index 07cf326fc1..ed6714b93c 100644 --- a/tests/integration/cmor/test_table.py +++ b/tests/integration/cmor/test_table.py @@ -1,7 +1,6 @@ """Integration tests for the variable_info module.""" import os -import unittest from pathlib import Path import pytest @@ -76,24 +75,25 @@ def test_update_cmor_facets_facet_not_in_table(mocker): assert facets == expected -class TestCMIP6Info(unittest.TestCase): - """Test for the CMIP6 info class.""" +class TestCMIP6Info: + """Tests for the CMIP6 info class.""" - def setUp(self): - self.variables_info = CMIP6Info( + @pytest.fixture + def variables_info(self) -> CMIP6Info: + return CMIP6Info( paths=[ Path("cmip6/Tables"), Path("cmip6-custom"), ], ) - def test_repr(self) -> None: + def test_repr(self, variables_info: CMIP6Info) -> None: builtin_tables_path = Path(esmvalcore.cmor.__file__).parent / "tables" expected_paths = [ builtin_tables_path / "cmip6" / "Tables", builtin_tables_path / "cmip6-custom", ] - result = repr(self.variables_info) + result = repr(variables_info) assert result.startswith( f"CMIP6Info(paths={expected_paths}, strict=True, alt_names=", ) @@ -106,71 +106,68 @@ def test_custom_tables_location(self): cmor_tables_path = os.path.abspath(cmor_tables_path) CMIP6Info(cmor_tables_path, default=None, strict=False) - def test_get_table_frequency(self): + def test_get_table_frequency(self, variables_info): """Test get table frequency.""" - self.assertEqual( - self.variables_info.get_table("Amon").frequency, - "mon", - ) - self.assertEqual(self.variables_info.get_table("day").frequency, "day") + assert variables_info.get_table("Amon").frequency == "mon" + assert variables_info.get_table("day").frequency == "day" - def test_get_variable_tas(self): + def test_get_variable_tas(self, variables_info): """Get tas variable.""" - var = self.variables_info.get_variable("Amon", "tas") - self.assertEqual(var.short_name, "tas") + var = variables_info.get_variable("Amon", "tas") + assert var.short_name == "tas" - def test_get_variable_from_alt_names(self): + def test_get_variable_from_alt_names(self, variables_info): """Get a variable from a known alt_names.""" - var = self.variables_info.get_variable("SImon", "sic") - self.assertEqual(var.short_name, "siconc") + var = variables_info.get_variable("SImon", "sic") + assert var.short_name == "siconc" - def test_get_variable_derived(self): + def test_get_variable_derived(self, variables_info): """Test that derived variable are looked up from other MIP tables.""" - var = self.variables_info.get_variable("3hr", "sfcWind", derived=True) - self.assertEqual(var.short_name, "sfcWind") + var = variables_info.get_variable("3hr", "sfcWind", derived=True) + assert var.short_name == "sfcWind" - def test_get_variable_from_custom(self): + def test_get_variable_from_custom(self, variables_info): """Get a variable from default.""" - self.variables_info.strict = False - var = self.variables_info.get_variable("Amon", "swcre") - self.assertEqual(var.short_name, "swcre") - self.assertEqual(var.frequency, "mon") + variables_info.strict = False + var = variables_info.get_variable("Amon", "swcre") + assert var.short_name == "swcre" + assert var.frequency == "mon" - var = self.variables_info.get_variable("day", "swcre") - self.assertEqual(var.short_name, "swcre") - self.assertEqual(var.frequency, "day") + var = variables_info.get_variable("day", "swcre") + assert var.short_name == "swcre" + assert var.frequency == "day" - def test_get_bad_variable(self): + def test_get_bad_variable(self, variables_info): """Get none if a variable is not in the given table.""" - self.assertIsNone(self.variables_info.get_variable("Omon", "ta")) + assert variables_info.get_variable("Omon", "ta") is None - def test_omon_ta_fail_if_strict(self): + def test_omon_ta_fail_if_strict(self, variables_info): """Get ta fails with Omon if strict.""" - self.assertIsNone(self.variables_info.get_variable("Omon", "ta")) + assert variables_info.get_variable("Omon", "ta") is None - def test_omon_ta_succes_if_strict(self): + def test_omon_ta_succes_if_strict(self, variables_info): """Get ta does not fail with AERMonZ if not strict.""" - self.variables_info.strict = False - var = self.variables_info.get_variable("Omon", "ta") - self.assertEqual(var.short_name, "ta") - self.assertEqual(var.frequency, "mon") + variables_info.strict = False + var = variables_info.get_variable("Omon", "ta") + assert var.short_name == "ta" + assert var.frequency == "mon" - def test_omon_toz_succes_if_strict(self): + def test_omon_toz_succes_if_strict(self, variables_info): """Get toz does not fail with Omon if not strict.""" - self.variables_info.strict = False - var = self.variables_info.get_variable("Omon", "toz") - self.assertEqual(var.short_name, "toz") - self.assertEqual(var.frequency, "mon") + variables_info.strict = False + var = variables_info.get_variable("Omon", "toz") + assert var.short_name == "toz" + assert var.frequency == "mon" - def test_get_institute_from_source(self): + def test_get_institute_from_source(self, variables_info): """Get institution for source ACCESS-CM2.""" - institute = self.variables_info.institutes["ACCESS-CM2"] - self.assertListEqual(institute, ["CSIRO-ARCCSS"]) + institute = variables_info.institutes["ACCESS-CM2"] + assert institute == ["CSIRO-ARCCSS"] - def test_get_activity_from_exp(self): + def test_get_activity_from_exp(self, variables_info): """Get activity for experiment 1pctCO2.""" - activity = self.variables_info.activities["1pctCO2"] - self.assertListEqual(activity, ["CMIP"]) + activity = variables_info.activities["1pctCO2"] + assert activity == ["CMIP"] def test_invalid_path(self) -> None: path = Path(__file__) / "path" / "does" / "not" / "exist" @@ -183,17 +180,19 @@ def test_invalid_paths(self) -> None: with pytest.raises(NotADirectoryError, match=str(path)): CMIP6Info(paths=[path]) + def test_invalid_file(self, tmp_path: Path) -> None: + invalid_file = tmp_path / "invalid.json" + invalid_file.touch() + with pytest.raises(ValueError): + CMIP6Info(paths=[tmp_path]) -class Testobs4mipsInfo(unittest.TestCase): - """Test for the obs$mips info class.""" - @classmethod - def setUpClass(cls): - """Set up tests. +class Testobs4mipsInfo: + """Tests for the obs4mips info class.""" - We read CMIP6Info once to keep tests times manageable - """ - cls.variables_info = Obs4MIPsInfo( + @pytest.fixture + def variables_info(self) -> Obs4MIPsInfo: + return Obs4MIPsInfo( paths=[ Path("obs4mips/Tables"), Path("cmip6-custom"), @@ -201,15 +200,9 @@ def setUpClass(cls): strict=False, ) - def setUp(self): - self.variables_info.strict = False - - def test_get_table_frequency(self): + def test_get_table_frequency(self, variables_info): """Test get table frequency.""" - self.assertEqual( - self.variables_info.get_table("monStderr").frequency, - "mon", - ) + assert variables_info.get_table("monStderr").frequency == "mon" def test_custom_tables_location(self): """Test constructor with custom tables location.""" @@ -218,75 +211,71 @@ def test_custom_tables_location(self): cmor_tables_path = os.path.abspath(cmor_tables_path) CMIP6Info(cmor_tables_path, None, True) - def test_get_variable_ndvistderr(self): + def test_get_variable_ndvistderr(self, variables_info): """Get ndviStderr variable. Note table name obs4MIPs_[mip] """ - var = self.variables_info.get_variable( + var = variables_info.get_variable( "obs4MIPs_monStderr", "ndviStderr", ) - self.assertEqual(var.short_name, "ndviStderr") - self.assertEqual(var.frequency, "mon") + assert var.short_name == "ndviStderr" + assert var.frequency == "mon" - def test_get_variable_hus(self): + def test_get_variable_hus(self, variables_info): """Get hus variable.""" - var = self.variables_info.get_variable("Amon", "hus") - self.assertEqual(var.short_name, "hus") - self.assertEqual(var.frequency, "mon") + var = variables_info.get_variable("Amon", "hus") + assert var.short_name == "hus" + assert var.frequency == "mon" - def test_get_variable_hus_default_prefix(self): + def test_get_variable_hus_default_prefix(self, variables_info): """Get hus variable.""" - var = self.variables_info.get_variable("Amon", "hus") - self.assertEqual(var.short_name, "hus") - self.assertEqual(var.frequency, "mon") + var = variables_info.get_variable("Amon", "hus") + assert var.short_name == "hus" + assert var.frequency == "mon" - def test_get_variable_from_custom(self): + def test_get_variable_from_custom(self, variables_info): """Get prStderr variable. Note table name obs4MIPs_[mip] """ - var = self.variables_info.get_variable( + var = variables_info.get_variable( "monStderr", "prStderr", ) - self.assertEqual(var.short_name, "prStderr") - self.assertEqual(var.frequency, "mon") + assert var.short_name == "prStderr" + assert var.frequency == "mon" - def test_get_variable_from_custom_deriving(self): + def test_get_variable_from_custom_deriving(self, variables_info): """Get a variable from default.""" - var = self.variables_info.get_variable( + var = variables_info.get_variable( "Amon", "swcre", derived=True, ) - self.assertEqual(var.short_name, "swcre") - self.assertEqual(var.frequency, "mon") + assert var.short_name == "swcre" + assert var.frequency == "mon" - var = self.variables_info.get_variable( + var = variables_info.get_variable( "Aday", "swcre", derived=True, ) - self.assertEqual(var.short_name, "swcre") - self.assertEqual(var.frequency, "day") + assert var.short_name == "swcre" + assert var.frequency == "day" - def test_get_bad_variable(self): + def test_get_bad_variable(self, variables_info): """Get none if a variable is not in the given table.""" - self.assertIsNone(self.variables_info.get_variable("Omon", "tras")) - + assert variables_info.get_variable("Omon", "tras") is None -class TestCMIP5Info(unittest.TestCase): - """Test for the CMIP5 info class.""" - @classmethod - def setUpClass(cls): - """Set up tests. +class TestCMIP5Info: + """Tests for the CMIP5 info class.""" - We read CMIP5Info once to keep testing times manageable - """ - cls.variables_info = CMIP5Info( + @pytest.fixture + def variables_info(self) -> CMIP5Info: + return CMIP5Info( paths=[ Path("cmip5/Tables"), Path("cmip5-custom"), @@ -294,9 +283,6 @@ def setUpClass(cls): strict=True, ) - def setUp(self): - self.variables_info.strict = True - def test_custom_tables_location(self): """Test constructor with custom tables location.""" cmor_path = os.path.dirname(os.path.realpath(esmvalcore.cmor.__file__)) @@ -304,82 +290,75 @@ def test_custom_tables_location(self): cmor_tables_path = os.path.abspath(cmor_tables_path) CMIP5Info(cmor_tables_path, None, True) - def test_get_variable_tas(self): + def test_get_variable_tas(self, variables_info): """Get tas variable.""" - var = self.variables_info.get_variable("Amon", "tas") - self.assertEqual(var.short_name, "tas") + var = variables_info.get_variable("Amon", "tas") + assert var.short_name == "tas" - def test_get_variable_zg(self): + def test_get_variable_zg(self, variables_info): """Get zg variable.""" - var = self.variables_info.get_variable("Amon", "zg") - self.assertEqual(var.short_name, "zg") - self.assertEqual( - var.coordinates["plevs"].requested, - [ - "100000.", - "92500.", - "85000.", - "70000.", - "60000.", - "50000.", - "40000.", - "30000.", - "25000.", - "20000.", - "15000.", - "10000.", - "7000.", - "5000.", - "3000.", - "2000.", - "1000.", - ], - ) + var = variables_info.get_variable("Amon", "zg") + assert var.short_name == "zg" + assert var.coordinates["plevs"].requested == [ + "100000.", + "92500.", + "85000.", + "70000.", + "60000.", + "50000.", + "40000.", + "30000.", + "25000.", + "20000.", + "15000.", + "10000.", + "7000.", + "5000.", + "3000.", + "2000.", + "1000.", + ] - def test_get_variable_from_custom(self): + def test_get_variable_from_custom(self, variables_info): """Get a variable from default.""" - self.variables_info.strict = False - var = self.variables_info.get_variable("Amon", "swcre") - self.assertEqual(var.short_name, "swcre") - self.assertEqual(var.frequency, "mon") + variables_info.strict = False + var = variables_info.get_variable("Amon", "swcre") + assert var.short_name == "swcre" + assert var.frequency == "mon" - var = self.variables_info.get_variable("day", "swcre") - self.assertEqual(var.short_name, "swcre") - self.assertEqual(var.frequency, "day") + var = variables_info.get_variable("day", "swcre") + assert var.short_name == "swcre" + assert var.frequency == "day" - def test_get_bad_variable(self): + def test_get_bad_variable(self, variables_info): """Get none if a variable is not in the given table.""" - self.assertIsNone(self.variables_info.get_variable("Omon", "tas")) + assert variables_info.get_variable("Omon", "tas") is None - def test_aermon_ta_fail_if_strict(self): + def test_aermon_ta_fail_if_strict(self, variables_info): """Get ta fails with AERMonZ if strict.""" - self.assertIsNone(self.variables_info.get_variable("Omon", "ta")) + assert variables_info.get_variable("Omon", "ta") is None - def test_aermon_ta_succes_if_strict(self): + def test_aermon_ta_succes_if_strict(self, variables_info): """Get ta does not fail with Omon if not strict.""" - self.variables_info.strict = False - var = self.variables_info.get_variable("Omon", "ta") - self.assertEqual(var.short_name, "ta") - self.assertEqual(var.frequency, "mon") + variables_info.strict = False + var = variables_info.get_variable("Omon", "ta") + assert var.short_name == "ta" + assert var.frequency == "mon" - def test_omon_toz_succes_if_strict(self): + def test_omon_toz_succes_if_strict(self, variables_info): """Get toz does not fail with Omon if not strict.""" - self.variables_info.strict = False - var = self.variables_info.get_variable("Omon", "toz") - self.assertEqual(var.short_name, "toz") - self.assertEqual(var.frequency, "mon") - + variables_info.strict = False + var = variables_info.get_variable("Omon", "toz") + assert var.short_name == "toz" + assert var.frequency == "mon" -class TestCMIP3Info(unittest.TestCase): - """Test for the CMIP5 info class.""" - @classmethod - def setUpClass(cls): - """Set up tests. +class TestCMIP3Info: + """Tests for the CMIP3 info class.""" - We read CMIP5Info once to keep testing times manageable - """ - cls.variables_info = CMIP3Info( + @pytest.fixture + def variables_info(self) -> CMIP3Info: + return CMIP3Info( paths=[ Path("cmip3/Tables"), Path("cmip5-custom"), @@ -387,9 +366,6 @@ def setUpClass(cls): strict=True, ) - def setUp(self): - self.variables_info.strict = True - def test_custom_tables_location(self): """Test constructor with custom tables location.""" cmor_path = os.path.dirname(os.path.realpath(esmvalcore.cmor.__file__)) @@ -397,82 +373,75 @@ def test_custom_tables_location(self): cmor_tables_path = os.path.abspath(cmor_tables_path) CMIP3Info(cmor_tables_path, None, True) - def test_get_variable_tas(self): + def test_get_variable_tas(self, variables_info): """Get tas variable.""" - var = self.variables_info.get_variable("A1", "tas") - self.assertEqual(var.short_name, "tas") + var = variables_info.get_variable("A1", "tas") + assert var.short_name == "tas" - def test_get_variable_zg(self): + def test_get_variable_zg(self, variables_info): """Get zg variable.""" - var = self.variables_info.get_variable("A1", "zg") - self.assertEqual(var.short_name, "zg") - self.assertEqual( - var.coordinates["pressure"].requested, - [ - "100000.", - "92500.", - "85000.", - "70000.", - "60000.", - "50000.", - "40000.", - "30000.", - "25000.", - "20000.", - "15000.", - "10000.", - "7000.", - "5000.", - "3000.", - "2000.", - "1000.", - ], - ) + var = variables_info.get_variable("A1", "zg") + assert var.short_name == "zg" + assert var.coordinates["pressure"].requested == [ + "100000.", + "92500.", + "85000.", + "70000.", + "60000.", + "50000.", + "40000.", + "30000.", + "25000.", + "20000.", + "15000.", + "10000.", + "7000.", + "5000.", + "3000.", + "2000.", + "1000.", + ] - def test_get_variable_from_custom(self): + def test_get_variable_from_custom(self, variables_info): """Get a variable from default.""" - self.variables_info.strict = False - var = self.variables_info.get_variable("A1", "swcre") - self.assertEqual(var.short_name, "swcre") - self.assertEqual(var.frequency, "") + variables_info.strict = False + var = variables_info.get_variable("A1", "swcre") + assert var.short_name == "swcre" + assert var.frequency == "" - var = self.variables_info.get_variable("day", "swcre") - self.assertEqual(var.short_name, "swcre") - self.assertEqual(var.frequency, "") + var = variables_info.get_variable("day", "swcre") + assert var.short_name == "swcre" + assert var.frequency == "" - def test_get_bad_variable(self): + def test_get_bad_variable(self, variables_info): """Get none if a variable is not in the given table.""" - self.assertIsNone(self.variables_info.get_variable("O1", "tas")) + assert variables_info.get_variable("O1", "tas") is None - def test_aermon_ta_fail_if_strict(self): + def test_aermon_ta_fail_if_strict(self, variables_info): """Get ta fails with AERMonZ if strict.""" - self.assertIsNone(self.variables_info.get_variable("O1", "ta")) + assert variables_info.get_variable("O1", "ta") is None - def test_aermon_ta_succes_if_strict(self): + def test_aermon_ta_succes_if_strict(self, variables_info): """Get ta does not fail with Omon if not strict.""" - self.variables_info.strict = False - var = self.variables_info.get_variable("O1", "ta") - self.assertEqual(var.short_name, "ta") - self.assertEqual(var.frequency, "") + variables_info.strict = False + var = variables_info.get_variable("O1", "ta") + assert var.short_name == "ta" + assert var.frequency == "" - def test_omon_toz_succes_if_strict(self): + def test_omon_toz_succes_if_strict(self, variables_info): """Get toz does not fail with Omon if not strict.""" - self.variables_info.strict = False - var = self.variables_info.get_variable("O1", "toz") - self.assertEqual(var.short_name, "toz") - self.assertEqual(var.frequency, "") + variables_info.strict = False + var = variables_info.get_variable("O1", "toz") + assert var.short_name == "toz" + assert var.frequency == "" -class TestCORDEXInfo(unittest.TestCase): - """Test for the CORDEX info class.""" +class TestCORDEXInfo: + """Tests for the CORDEX info class.""" - @classmethod - def setUpClass(cls): - """Set up tests. - - We read CORDEX once to keep testing times manageable - """ - cls.variables_info = CMIP5Info( + @pytest.fixture + def variables_info(self) -> CMIP5Info: + return CMIP5Info( paths=[ Path("cordex/Tables"), Path("cmip5-custom"), @@ -485,28 +454,24 @@ def test_custom_tables_location(self): cmor_tables_path = os.path.join(cmor_path, "tables", "cordex") CMIP5Info(cmor_tables_path) - def test_get_variable_tas(self): + def test_get_variable_tas(self, variables_info): """Get tas variable.""" - var = self.variables_info.get_variable("mon", "tas") - self.assertEqual(var.short_name, "tas") + var = variables_info.get_variable("mon", "tas") + assert var.short_name == "tas" - def test_get_bad_variable(self): + def test_get_bad_variable(self, variables_info): """Get none if a variable is not in the given table.""" - self.assertIsNone(self.variables_info.get_variable("Omon", "tas")) + assert variables_info.get_variable("Omon", "tas") is None -class TestCustomInfo(unittest.TestCase): - """Test for the custom info class.""" +class TestCustomInfo: + """Tests for the custom info class.""" - @classmethod - def setUpClass(cls): - """Set up tests. + @pytest.fixture + def variables_info(self) -> CustomInfo: + return CustomInfo() - We read CMIP5Info once to keep testing times manageable - """ - cls.variables_info = CustomInfo() - - def test_custom_tables_default_location(self): + def test_custom_tables_default_location(self, variables_info): """Test constructor with default tables location.""" custom_info = CustomInfo() expected_cmor_folder = os.path.join( @@ -515,10 +480,10 @@ def test_custom_tables_default_location(self): "cmip5-custom", ) assert custom_info.paths == (Path(expected_cmor_folder),) - self.assertTrue(custom_info.tables["custom"]) - self.assertTrue(custom_info.coords) + assert custom_info.tables["custom"] + assert custom_info.coords - def test_custom_tables_location(self): + def test_custom_tables_location(self, variables_info): """Test constructor with custom tables location.""" cmor_path = os.path.dirname(os.path.realpath(esmvalcore.cmor.__file__)) default_cmor_tables_path = os.path.join( @@ -535,72 +500,69 @@ def test_custom_tables_location(self): Path(default_cmor_tables_path), Path(cmor_tables_path), ) - self.assertTrue(custom_info.tables["custom"]) - self.assertTrue(custom_info.coords) + assert custom_info.tables["custom"] + assert custom_info.coords def test_custom_tables_invalid_location(self): """Test constructor with invalid custom tables location.""" - with self.assertRaises(ValueError): + with pytest.raises(ValueError): CustomInfo("this_file_does_not_exist.dat") - def test_get_variable_netcre(self): + def test_get_variable_netcre(self, variables_info): """Get tas variable.""" CustomInfo() - var = self.variables_info.get_variable("Amon", "netcre") - self.assertEqual(var.short_name, "netcre") + var = variables_info.get_variable("Amon", "netcre") + assert var.short_name == "netcre" - def test_get_bad_variable(self): + def test_get_bad_variable(self, variables_info): """Get none if a variable is not in the given table.""" - self.assertIsNone(self.variables_info.get_variable("Omon", "badvar")) + assert variables_info.get_variable("Omon", "badvar") is None - def test_get_variable_tasconf5(self): + def test_get_variable_tasconf5(self, variables_info): """Get tas variable.""" CustomInfo() - var = self.variables_info.get_variable("Amon", "tasConf5") - self.assertEqual(var.short_name, "tasConf5") - self.assertEqual( - var.long_name, - "Near-Surface Air Temperature Uncertainty Range", + var = variables_info.get_variable("Amon", "tasConf5") + assert var.short_name == "tasConf5" + assert ( + var.long_name == "Near-Surface Air Temperature Uncertainty Range" ) - self.assertEqual(var.units, "K") + assert var.units == "K" - def test_get_variable_tasconf95(self): + def test_get_variable_tasconf95(self, variables_info): """Get tas variable.""" CustomInfo() - var = self.variables_info.get_variable("Amon", "tasConf95") - self.assertEqual(var.short_name, "tasConf95") - self.assertEqual( - var.long_name, - "Near-Surface Air Temperature Uncertainty Range", + var = variables_info.get_variable("Amon", "tasConf95") + assert var.short_name == "tasConf95" + assert ( + var.long_name == "Near-Surface Air Temperature Uncertainty Range" ) - self.assertEqual(var.units, "K") + assert var.units == "K" - def test_get_variable_tasaga(self): + def test_get_variable_tasaga(self, variables_info): """Get tas variable.""" CustomInfo() - var = self.variables_info.get_variable("Amon", "tasaga") - self.assertEqual(var.short_name, "tasaga") - self.assertEqual( - var.long_name, - "Global-mean Near-Surface Air Temperature Anomaly", + var = variables_info.get_variable("Amon", "tasaga") + assert var.short_name == "tasaga" + assert ( + var.long_name == "Global-mean Near-Surface Air Temperature Anomaly" ) - self.assertEqual(var.units, "K") + assert var.units == "K" - def test_get_variable_ch4s(self): + def test_get_variable_ch4s(self, variables_info): """Get ch4s variable.""" CustomInfo() - var = self.variables_info.get_variable("Amon", "ch4s") - self.assertEqual(var.short_name, "ch4s") - self.assertEqual(var.long_name, "Atmosphere CH4 surface") - self.assertEqual(var.units, "1e-09") + var = variables_info.get_variable("Amon", "ch4s") + assert var.short_name == "ch4s" + assert var.long_name == "Atmosphere CH4 surface" + assert var.units == "1e-09" - def test_get_variable_tosstderr(self): + def test_get_variable_tosstderr(self, variables_info): """Get tosStderr variable.""" CustomInfo() - var = self.variables_info.get_variable("Omon", "tosStderr") - self.assertEqual(var.short_name, "tosStderr") - self.assertEqual(var.long_name, "Sea Surface Temperature Error") - self.assertEqual(var.units, "K") + var = variables_info.get_variable("Omon", "tosStderr") + assert var.short_name == "tosStderr" + assert var.long_name == "Sea Surface Temperature Error" + assert var.units == "K" @pytest.mark.parametrize( From 067c6f046ec4077bb41c34cf51af2c67df72416c Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Thu, 29 Jan 2026 22:02:55 +0100 Subject: [PATCH 19/47] Add more tests --- tests/integration/cmor/test_table.py | 40 ++++++++++++++++++++++++++++ tests/unit/cmor/test_table.py | 5 ++++ 2 files changed, 45 insertions(+) diff --git a/tests/integration/cmor/test_table.py b/tests/integration/cmor/test_table.py index ed6714b93c..112030ed77 100644 --- a/tests/integration/cmor/test_table.py +++ b/tests/integration/cmor/test_table.py @@ -1,5 +1,7 @@ """Integration tests for the variable_info module.""" +from __future__ import annotations + import os from pathlib import Path @@ -12,7 +14,9 @@ CMIP5Info, CMIP6Info, CustomInfo, + NoInfo, Obs4MIPsInfo, + VariableInfo, _get_branding_suffixes, _get_mips, _update_cmor_facets, @@ -180,6 +184,10 @@ def test_invalid_paths(self) -> None: with pytest.raises(NotADirectoryError, match=str(path)): CMIP6Info(paths=[path]) + def test_no_tables_in_path(self, tmp_path: Path) -> None: + with pytest.raises(ValueError): + CMIP6Info(paths=[tmp_path]) + def test_invalid_file(self, tmp_path: Path) -> None: invalid_file = tmp_path / "invalid.json" invalid_file.touch() @@ -352,6 +360,12 @@ def test_omon_toz_succes_if_strict(self, variables_info): assert var.short_name == "toz" assert var.frequency == "mon" + def test_invalid_file(self, tmp_path: Path) -> None: + invalid_file = tmp_path / "invalid" + invalid_file.write_text("invalid content", encoding="utf-8") + with pytest.raises(ValueError): + CMIP5Info(paths=[tmp_path]) + class TestCMIP3Info: """Tests for the CMIP3 info class.""" @@ -471,6 +485,14 @@ class TestCustomInfo: def variables_info(self) -> CustomInfo: return CustomInfo() + def test_repr(self, variables_info: CustomInfo) -> None: + builtin_tables_path = Path(esmvalcore.cmor.__file__).parent / "tables" + expected_paths = [ + builtin_tables_path / "cmip5-custom", + ] + result = repr(variables_info) + assert result == f"CustomInfo(paths={expected_paths})" + def test_custom_tables_default_location(self, variables_info): """Test constructor with default tables location.""" custom_info = CustomInfo() @@ -565,6 +587,24 @@ def test_get_variable_tosstderr(self, variables_info): assert var.units == "K" +class TestNoInfo: + """Tests for the no info class.""" + + @pytest.fixture + def variables_info(self) -> NoInfo: + return NoInfo() + + def test_repr(self, variables_info: NoInfo) -> None: + result = repr(variables_info) + assert result == "NoInfo()" + + def test_get_variable_tas(self, variables_info: NoInfo) -> None: + """Get tas variable.""" + var = variables_info.get_variable("Amon", "tas") + assert isinstance(var, VariableInfo) + assert var.short_name == "tas" + + @pytest.mark.parametrize( ("project", "mip", "short_name", "frequency"), [ diff --git a/tests/unit/cmor/test_table.py b/tests/unit/cmor/test_table.py index 2a12d18599..81a286a95a 100644 --- a/tests/unit/cmor/test_table.py +++ b/tests/unit/cmor/test_table.py @@ -18,6 +18,11 @@ def setUp(self): "dim2": CoordinateInfo("dim2"), } + def test_repr(self) -> None: + """Test __repr__.""" + result = repr(self.info) + assert result == "" + def test_constructor(self): """Test basic constructor.""" self.assertEqual("table_type", self.info.table_type) From 69487f0cc8bf5f1e7fc8b4d8f016aec649732f37 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Thu, 29 Jan 2026 22:22:20 +0100 Subject: [PATCH 20/47] Add more tests --- tests/integration/cmor/test_table.py | 45 +++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/tests/integration/cmor/test_table.py b/tests/integration/cmor/test_table.py index 112030ed77..bf69237587 100644 --- a/tests/integration/cmor/test_table.py +++ b/tests/integration/cmor/test_table.py @@ -2,6 +2,7 @@ from __future__ import annotations +import logging import os from pathlib import Path @@ -190,10 +191,31 @@ def test_no_tables_in_path(self, tmp_path: Path) -> None: def test_invalid_file(self, tmp_path: Path) -> None: invalid_file = tmp_path / "invalid.json" - invalid_file.touch() + invalid_file.write_text("invalid content", encoding="utf-8") with pytest.raises(ValueError): CMIP6Info(paths=[tmp_path]) + def test_invalid_file_logged( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + ) -> None: + # attach the root logger handler added by caplog + monkeypatch.setattr( + esmvalcore.cmor.table.logger, + "handlers", + logging.getLogger().handlers, + ) + invalid_file = tmp_path / "invalid.json" + invalid_file.write_text("invalid content", encoding="utf-8") + with pytest.raises(ValueError): + CMIP6Info(paths=[tmp_path]) + assert ( + f"Exception raised when loading {tmp_path}/invalid.json" + in caplog.messages + ) + class Testobs4mipsInfo: """Tests for the obs4mips info class.""" @@ -366,6 +388,27 @@ def test_invalid_file(self, tmp_path: Path) -> None: with pytest.raises(ValueError): CMIP5Info(paths=[tmp_path]) + def test_invalid_file_logged( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + ) -> None: + # attach the root logger handler added by caplog + monkeypatch.setattr( + esmvalcore.cmor.table.logger, + "handlers", + logging.getLogger().handlers, + ) + invalid_file = tmp_path / "invalid" + invalid_file.write_text("invalid content", encoding="utf-8") + with pytest.raises(ValueError): + CMIP5Info(paths=[tmp_path]) + assert ( + f"Exception raised when loading {tmp_path}/invalid" + in caplog.messages + ) + class TestCMIP3Info: """Tests for the CMIP3 info class.""" From 8c13364412f6434c0116e6273533accb517b015d Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Fri, 30 Jan 2026 08:52:58 +0100 Subject: [PATCH 21/47] Add copilot instructions --- .github/instructions/*.instructions.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 .github/instructions/*.instructions.md diff --git a/.github/instructions/*.instructions.md b/.github/instructions/*.instructions.md new file mode 100644 index 0000000000..309a9c88a5 --- /dev/null +++ b/.github/instructions/*.instructions.md @@ -0,0 +1 @@ +Guidelines are available in the file doc/contributing.rst From dd2f560248cda678d63174cd106816c5a5e7c5a7 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Fri, 30 Jan 2026 09:23:47 +0100 Subject: [PATCH 22/47] Implement suggestions from code review --- doc/quickstart/configure.rst | 2 +- esmvalcore/cmor/table.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/quickstart/configure.rst b/doc/quickstart/configure.rst index 5e462be3b8..24cab18e08 100644 --- a/doc/quickstart/configure.rst +++ b/doc/quickstart/configure.rst @@ -748,7 +748,7 @@ for loading and saving the data in the :ref:`recipe ` or .. warning:: While it is possible to work with datasets that are not described in a CMOR - table,the :ref:`preprocessor functions ` and + table, the :ref:`preprocessor functions ` and :ref:`diagnostics ` have been designed to work with CMORized data and may not work as expected with non-CMORized data. diff --git a/esmvalcore/cmor/table.py b/esmvalcore/cmor/table.py index 5dd58659d2..06cd431cb4 100644 --- a/esmvalcore/cmor/table.py +++ b/esmvalcore/cmor/table.py @@ -1569,7 +1569,7 @@ class NoInfo(InfoBase): """Table that can be used for projects that do not provide a CMOR table.""" def __init__(self) -> None: - pass + super().__init__() def __repr__(self) -> str: return f"{self.__class__.__name__}()" From 08924a9c82d5d4a9121d26d64afdeead3d348615 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Fri, 30 Jan 2026 09:29:25 +0100 Subject: [PATCH 23/47] Address more review comments --- tests/integration/cmor/test_read_cmor_tables.py | 9 +++++---- tests/unit/preprocessor/test_configuration.py | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/integration/cmor/test_read_cmor_tables.py b/tests/integration/cmor/test_read_cmor_tables.py index efc3e1175c..caee63dd5d 100644 --- a/tests/integration/cmor/test_read_cmor_tables.py +++ b/tests/integration/cmor/test_read_cmor_tables.py @@ -7,10 +7,11 @@ import pytest import yaml -import esmvalcore.cmor.table from esmvalcore.cmor.table import ( + _TABLE_CACHE, CMOR_TABLES, VariableInfo, + clear_table_cache, get_tables, read_cmor_tables, ) @@ -171,9 +172,9 @@ def test_get_tables_invalid_type( def test_clear_table_cache(session: Session) -> None: - assert esmvalcore.cmor.table._TABLE_CACHE - esmvalcore.cmor.table.clear_table_cache() - assert not esmvalcore.cmor.table._TABLE_CACHE + assert _TABLE_CACHE + clear_table_cache() + assert not _TABLE_CACHE CMOR_NEWVAR_ENTRY = dedent( diff --git a/tests/unit/preprocessor/test_configuration.py b/tests/unit/preprocessor/test_configuration.py index 539f237873..8b79c149ab 100644 --- a/tests/unit/preprocessor/test_configuration.py +++ b/tests/unit/preprocessor/test_configuration.py @@ -8,7 +8,7 @@ import pytest import esmvalcore -import esmvalcore.config +from esmvalcore.config import CFG from esmvalcore.dataset import Dataset from esmvalcore.exceptions import RecipeError from esmvalcore.preprocessor import ( @@ -155,7 +155,7 @@ def test_get_preprocessor_filename_falls_back_to_config_developer( ) -> None: """Test the function `_get_preprocessor_filename`.""" monkeypatch.setitem( - esmvalcore.config.CFG, + CFG, "config_developer_file", Path(esmvalcore.__path__[0], "config-developer.yml"), ) From 7a6c11a24ee301dc6dcf8cfa916720b0cb92a49b Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Fri, 30 Jan 2026 10:02:27 +0100 Subject: [PATCH 24/47] Fix typo in link Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- doc/reference/cmor_tables.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/reference/cmor_tables.rst b/doc/reference/cmor_tables.rst index b872817e13..20b15e3d55 100644 --- a/doc/reference/cmor_tables.rst +++ b/doc/reference/cmor_tables.rst @@ -52,7 +52,7 @@ select one of the near-surface air temperature variables in the CMIP7 atmos tabl .. literalinclude:: ../../esmvalcore/cmor/tables/cmip7/tables/CMIP7_atmos.json :start-at: "tas_tavg-h2m-hxy-u": { :end-at: }, - :caption: One of the ``tas`` variable definitions in the CMIP7 atmos table at `esmvalcore/cmor/tables/cmip7/tables/CMIP7_atmos.json `__. + :caption: One of the ``tas`` variable definitions in the CMIP7 atmos table at `esmvalcore/cmor/tables/cmip7/tables/CMIP7_atmos.json `__. For other projects, the facet ``branding_suffix`` can also be used to distinguish between variables from the same CMOR table that share the same ``short_name``, From aff84c87cface5258316d0d77118e771342ac5c7 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Fri, 30 Jan 2026 09:59:42 +0100 Subject: [PATCH 25/47] Deprecate esmvalcore.cmor.table.CMOR_TABLES becuase of #2954 --- esmvalcore/cmor/table.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/esmvalcore/cmor/table.py b/esmvalcore/cmor/table.py index 06cd431cb4..225d7027bd 100644 --- a/esmvalcore/cmor/table.py +++ b/esmvalcore/cmor/table.py @@ -33,6 +33,11 @@ CMOR_TABLES: dict[str, InfoBase] = {} """dict of str, obj: CMOR info objects. +.. deprecated:: 2.14.0 + The global ``CMOR_TABLES`` dictionary is deprecated and will be removed in + ESMValCore v2.16.0. Please use :func:`~esmvalcore.cmor.table.get_tables` + to access the CMOR tables instead. + .. note:: If this dictionary is empty, it can be populated by loading the global configuration by importing the :mod:`esmvalcore.config` module. From 4199ca54ef2ab57f4eacaa7bd122515d2cb1af23 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Fri, 30 Jan 2026 10:36:00 +0100 Subject: [PATCH 26/47] Small improvements --- esmvalcore/config/_config_validators.py | 2 +- tests/integration/cmor/test_table.py | 5 +++++ tests/unit/config/test_config.py | 8 ++------ 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/esmvalcore/config/_config_validators.py b/esmvalcore/config/_config_validators.py index 6cf3a05b83..e2b9da448c 100644 --- a/esmvalcore/config/_config_validators.py +++ b/esmvalcore/config/_config_validators.py @@ -380,7 +380,7 @@ def validate_cmor_tables(value: dict) -> None: # # Relying on global state is not nice, preferably we should get rid of any # global state except the defaults for starting a new session in - # `esmvalcore.config.CFG`. + # `esmvalcore.config.CFG`. https://github.com/esMValGroup/esMValCore/issues/2954 # # Having side effects when updating an `esmvalcore.config.Session` object # that changes global state of the `esmvalcore` package is nasty and should diff --git a/tests/integration/cmor/test_table.py b/tests/integration/cmor/test_table.py index bf69237587..21e68f75d1 100644 --- a/tests/integration/cmor/test_table.py +++ b/tests/integration/cmor/test_table.py @@ -90,6 +90,11 @@ def variables_info(self) -> CMIP6Info: Path("cmip6/Tables"), Path("cmip6-custom"), ], + strict=True, + alt_names=[ + ["sic", "siconc"], + ["tro3", "o3"], + ], ) def test_repr(self, variables_info: CMIP6Info) -> None: diff --git a/tests/unit/config/test_config.py b/tests/unit/config/test_config.py index 4c8aed9775..9a7c9b8775 100644 --- a/tests/unit/config/test_config.py +++ b/tests/unit/config/test_config.py @@ -162,13 +162,9 @@ def test_get_project_config(mocker): def test_load_default_config(cfg_default, monkeypatch): """Test that the default configuration can be loaded.""" root_path = importlib_files("esmvalcore") - config_dir = root_path / "config" / "configurations" / "defaults" + default_config_dir = root_path / "config" / "configurations" / "defaults" default_project_settings = dask.config.collect( - paths=[str(p) for p in config_dir.glob("extra_facets_*.yml")] - + [ - str(config_dir / "preprocessor_filename_template.yml"), - str(config_dir / "cmor_tables.yml"), - ], + paths=[str(p) for p in default_config_dir.glob("*.yml")], env={}, )["projects"] From fda20003b75607a8733beb34d06e68a13340c999 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Fri, 30 Jan 2026 10:48:57 +0100 Subject: [PATCH 27/47] Improve AI instructions --- .github/instructions/*.instructions.md | 2 +- .gitignore | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) mode change 100644 => 120000 .github/instructions/*.instructions.md diff --git a/.github/instructions/*.instructions.md b/.github/instructions/*.instructions.md deleted file mode 100644 index 309a9c88a5..0000000000 --- a/.github/instructions/*.instructions.md +++ /dev/null @@ -1 +0,0 @@ -Guidelines are available in the file doc/contributing.rst diff --git a/.github/instructions/*.instructions.md b/.github/instructions/*.instructions.md new file mode 120000 index 0000000000..fc66fcc90c --- /dev/null +++ b/.github/instructions/*.instructions.md @@ -0,0 +1 @@ +doc/contributing.rst \ No newline at end of file diff --git a/.gitignore b/.gitignore index ee821184de..1b9d52962e 100644 --- a/.gitignore +++ b/.gitignore @@ -109,3 +109,7 @@ doc/_sidebar.rst.inc # ESMF log files *.ESMF_LogFile + + +#Ignore vscode AI rules +.github/instructions/codacy.instructions.md From 5519c3f029b23c74358cdbfbc2ef2952ded319cd Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Mon, 2 Feb 2026 16:58:51 +0100 Subject: [PATCH 28/47] Improve type hint --- esmvalcore/cmor/_fixes/fix.py | 2 +- esmvalcore/cmor/_utils.py | 2 +- esmvalcore/cmor/check.py | 10 +++++++--- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/esmvalcore/cmor/_fixes/fix.py b/esmvalcore/cmor/_fixes/fix.py index 26c587f9ed..7f2bfc216e 100644 --- a/esmvalcore/cmor/_fixes/fix.py +++ b/esmvalcore/cmor/_fixes/fix.py @@ -572,7 +572,7 @@ def _fix_alternative_generic_level_coords(self, cube: Cube) -> Cube: # Make sure to update variable information with actual generic # level coordinate if one has been found; this is necessary for # subsequent fixes - if standard_name: + if name is not None: new_generic_level_coord = _get_new_generic_level_coord( self.vardef, cmor_coord, diff --git a/esmvalcore/cmor/_utils.py b/esmvalcore/cmor/_utils.py index f939caadf3..2272a43e38 100644 --- a/esmvalcore/cmor/_utils.py +++ b/esmvalcore/cmor/_utils.py @@ -130,7 +130,7 @@ def _get_new_generic_level_coord( var_info: VariableInfo, generic_level_coord: CoordinateInfo, generic_level_coord_name: str, - new_coord_name: str | None, + new_coord_name: str, ) -> CoordinateInfo: """Get new generic level coordinate. diff --git a/esmvalcore/cmor/check.py b/esmvalcore/cmor/check.py index 3e27da363d..21121be515 100644 --- a/esmvalcore/cmor/check.py +++ b/esmvalcore/cmor/check.py @@ -425,14 +425,18 @@ def _check_dim_names(self): "exist", ) - def _check_generic_level_dim_names(self, key, coordinate): + def _check_generic_level_dim_names( + self, + key: str, + coordinate: CoordinateInfo, + ) -> None: """Check name of generic level coordinate.""" if coordinate.generic_lev_coords: - (standard_name, out_name, name) = _get_generic_lev_coord_names( + (_, out_name, name) = _get_generic_lev_coord_names( self._cube, coordinate, ) - if standard_name: + if name is not None: if not out_name: self.report_error( f"Generic level coordinate {key} has wrong var_name.", From a18ee08e64f381173ad7887bda4da1ddccd6eed4 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Wed, 4 Feb 2026 09:46:40 +0100 Subject: [PATCH 29/47] Add punctuation Co-authored-by: Valeriu Predoi --- doc/develop/fixing_data.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/develop/fixing_data.rst b/doc/develop/fixing_data.rst index 350325fb98..06da1aa751 100644 --- a/doc/develop/fixing_data.rst +++ b/doc/develop/fixing_data.rst @@ -459,7 +459,7 @@ To allow ESMValCore to locate the data files, use the following steps: :start-at: ICON: :end-at: strict: false - To find your ICON data that is for example located in files like + To find your ICON data that is, for example, located in files like ``~/climate_data/amip/amip_atm_2d_ml_20000101T000000Z.nc``, use the following dataset entry in your recipe: From 2b711c9510bee8995322bc71c7307be2f49b510b Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Wed, 4 Feb 2026 10:07:35 +0100 Subject: [PATCH 30/47] Clarify data requirements for non-cmorized data --- doc/contributing.rst | 7 +++++-- doc/develop/fixing_data.rst | 6 ++++++ doc/quickstart/configure.rst | 6 ++++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/doc/contributing.rst b/doc/contributing.rst index 3c5a3e8ca6..cc5a33956c 100644 --- a/doc/contributing.rst +++ b/doc/contributing.rst @@ -72,7 +72,9 @@ Please keep the following considerations in mind when programming: what exactly is being changed. - :ref:`preprocessor_functions` are Python functions (and not classes) so they are easy to understand and implement for scientific contributors. -- No additional CMOR checks should be implemented inside preprocessor functions. +- No additional CMOR checks should be implemented inside preprocessor functions, + though it is our ambition to make preprocessor functions work with any data that + follows the `CF Conventions `_. The input cube is fixed and confirmed to follow the specification in `esmvalcore/cmor/tables `__ before applying any other preprocessor functions. @@ -82,7 +84,8 @@ Please keep the following considerations in mind when programming: recipe to the relevant CMOR table. - The ESMValCore package is based on :ref:`iris `. Preprocessor functions should preferably be small and just call the relevant - iris code. + iris code. This automatically ensures that the preprocessor functions work with + data that follows the CF Conventions. Code that is more involved and more broadly applicable than just in the ESMValCore, should be implemented in iris instead. - Any settings in the recipe that can be checked before loading the data should diff --git a/doc/develop/fixing_data.rst b/doc/develop/fixing_data.rst index 06da1aa751..6f6504bd6f 100644 --- a/doc/develop/fixing_data.rst +++ b/doc/develop/fixing_data.rst @@ -363,6 +363,12 @@ below from the lowest level of strictness to the highest: :ref:`diagnostics ` have been designed to work with CMORized data and may not work as expected with non-CMORized data. + Our ambition is that :ref:`preprocessor functions ` support + data that follows the + `CF Conventions `_, while + :ref:`diagnostics ` are only expected to work with + CMORized data. + .. _add_new_fix_native_datasets: Add support for new native datasets diff --git a/doc/quickstart/configure.rst b/doc/quickstart/configure.rst index 24cab18e08..462e8e1e5c 100644 --- a/doc/quickstart/configure.rst +++ b/doc/quickstart/configure.rst @@ -752,6 +752,12 @@ for loading and saving the data in the :ref:`recipe ` or :ref:`diagnostics ` have been designed to work with CMORized data and may not work as expected with non-CMORized data. + Our ambition is that :ref:`preprocessor functions ` support + data that follows the + `CF Conventions `_, while + :ref:`diagnostics ` are only expected to work with + CMORized data. + .. _config-data-sources: Data sources From e0edb85759b3de79192ec651f45fa10ce8ce1928 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Wed, 4 Feb 2026 10:09:11 +0100 Subject: [PATCH 31/47] Spelling Co-authored-by: Valeriu Predoi --- doc/quickstart/configure.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/quickstart/configure.rst b/doc/quickstart/configure.rst index 462e8e1e5c..7068564005 100644 --- a/doc/quickstart/configure.rst +++ b/doc/quickstart/configure.rst @@ -1242,7 +1242,7 @@ following mapping applies: Because it is now possible to configure multiple paths to directories containing CMOR tables per project, the ``custom`` project for specifying additional custom -CMOR tables is no longer needed or supported. +CMOR tables is no longer needed nor supported. Example 1: A config-developer.yml file specifying different CMIP6 CMOR tables than the default ones, augmented by the default custom CMOR tables: From daec42645c070199ddf05aa9fd8c67b6603a516b Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Thu, 5 Feb 2026 12:24:49 +0100 Subject: [PATCH 32/47] Add examples to docstrings --- esmvalcore/cmor/table.py | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/esmvalcore/cmor/table.py b/esmvalcore/cmor/table.py index 225d7027bd..da17063c04 100644 --- a/esmvalcore/cmor/table.py +++ b/esmvalcore/cmor/table.py @@ -487,12 +487,15 @@ def get_variable( ---------- table_name: Table name, i.e., the ``mip`` in the :ref:`recipe ` or - :class:`~esmvalcore.dataset.Dataset`. + :class:`~esmvalcore.dataset.Dataset`, e.g. ``"Omon"`` for CMIP6 or + ``"ocean"`` for CMIP7. short_name: - Variable's short name. + Variable's short name, e.g. "tos" for sea surface temperature. branding_suffix: A suffix that will be appended to ``short_name`` when looking up the - variable in the CMOR table. + variable in the CMOR table, e.g. ``"tavg-u-hxy-sea"`` for the + temporal average at an undefined vertical level on a horizontal grid + where non-sea points are masked. derived: Variable is derived. Information retrieval for derived variables always looks in the default tables (usually, the custom tables) if @@ -1525,13 +1528,17 @@ def get_variable( Parameters ---------- - table: - Table name. Ignored for custom tables. + table_name: + Table name, i.e., the ``mip`` in the :ref:`recipe ` or + :class:`~esmvalcore.dataset.Dataset`, e.g. ``"Omon"`` for CMIP6 or + ``"ocean"`` for CMIP7. short_name: - Variable's short name. + Variable's short name, e.g. "tos" for sea surface temperature. branding_suffix: A suffix that will be appended to ``short_name`` when looking up the - variable in the CMOR table. Ignored for custom tables. + variable in the CMOR table, e.g. ``"tavg-u-hxy-sea"`` for the + temporal average at an undefined vertical level on a horizontal grid + where non-sea points are masked. derived: Variable is derived. Info retrieval for derived variables always looks on the default tables if variable is not found in the @@ -1593,12 +1600,15 @@ def get_variable( ---------- table_name: Table name, i.e., the ``mip`` in the :ref:`recipe ` or - :class:`~esmvalcore.dataset.Dataset`. + :class:`~esmvalcore.dataset.Dataset`, e.g. ``"Omon"`` for CMIP6 or + ``"ocean"`` for CMIP7. short_name: - Variable's short name. + Variable's short name, e.g. "tos" for sea surface temperature. branding_suffix: A suffix that will be appended to ``short_name`` when looking up the - variable in the CMOR table. + variable in the CMOR table, e.g. ``"tavg-u-hxy-sea"`` for the + temporal average at an undefined vertical level on a horizontal grid + where non-sea points are masked. derived: Variable is derived. Information retrieval for derived variables always looks in the default tables (usually, the custom tables) if From fea3efd75e8709fcd05536e3edd83f0a8fe4c0f7 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Thu, 5 Feb 2026 12:31:15 +0100 Subject: [PATCH 33/47] Add link for branded variable background --- esmvalcore/cmor/table.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/esmvalcore/cmor/table.py b/esmvalcore/cmor/table.py index da17063c04..484859b5cb 100644 --- a/esmvalcore/cmor/table.py +++ b/esmvalcore/cmor/table.py @@ -493,9 +493,10 @@ def get_variable( Variable's short name, e.g. "tos" for sea surface temperature. branding_suffix: A suffix that will be appended to ``short_name`` when looking up the - variable in the CMOR table, e.g. ``"tavg-u-hxy-sea"`` for the - temporal average at an undefined vertical level on a horizontal grid - where non-sea points are masked. + variable in the CMOR table, e.g. for + `CMIP7 `__, + ``"tavg-u-hxy-sea"`` defines the temporal average at an undefined + vertical level on a horizontal grid where non-sea points are masked. derived: Variable is derived. Information retrieval for derived variables always looks in the default tables (usually, the custom tables) if @@ -1536,9 +1537,11 @@ def get_variable( Variable's short name, e.g. "tos" for sea surface temperature. branding_suffix: A suffix that will be appended to ``short_name`` when looking up the - variable in the CMOR table, e.g. ``"tavg-u-hxy-sea"`` for the - temporal average at an undefined vertical level on a horizontal grid - where non-sea points are masked. + variable in the CMOR table, e.g. a + `CMIP7 branding suffix `__, + could be ``"tavg-u-hxy-sea"``, which defines the temporal average + at an undefined vertical level on a horizontal grid where non-sea + points are masked. derived: Variable is derived. Info retrieval for derived variables always looks on the default tables if variable is not found in the @@ -1606,9 +1609,11 @@ def get_variable( Variable's short name, e.g. "tos" for sea surface temperature. branding_suffix: A suffix that will be appended to ``short_name`` when looking up the - variable in the CMOR table, e.g. ``"tavg-u-hxy-sea"`` for the - temporal average at an undefined vertical level on a horizontal grid - where non-sea points are masked. + variable in the CMOR table, e.g. a + `CMIP7 branding suffix `__, + could be ``"tavg-u-hxy-sea"``, which defines the temporal average + at an undefined vertical level on a horizontal grid where non-sea + points are masked. derived: Variable is derived. Information retrieval for derived variables always looks in the default tables (usually, the custom tables) if From 403bd623bc642af78fac1c79847ebf8fa3506683 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Thu, 5 Feb 2026 12:44:41 +0100 Subject: [PATCH 34/47] Fix docs --- esmvalcore/cmor/table.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/esmvalcore/cmor/table.py b/esmvalcore/cmor/table.py index 484859b5cb..b6e6edbb1e 100644 --- a/esmvalcore/cmor/table.py +++ b/esmvalcore/cmor/table.py @@ -490,13 +490,14 @@ def get_variable( :class:`~esmvalcore.dataset.Dataset`, e.g. ``"Omon"`` for CMIP6 or ``"ocean"`` for CMIP7. short_name: - Variable's short name, e.g. "tos" for sea surface temperature. + Variable's short name, e.g. ``"tos"`` for sea surface temperature. branding_suffix: A suffix that will be appended to ``short_name`` when looking up the - variable in the CMOR table, e.g. for - `CMIP7 `__, - ``"tavg-u-hxy-sea"`` defines the temporal average at an undefined - vertical level on a horizontal grid where non-sea points are masked. + variable in the CMOR table, e.g. a + `CMIP7 branding suffix `__, + could be ``"tavg-u-hxy-sea"``, which defines the temporal average + at an undefined vertical level on a horizontal grid where non-sea + points are masked. derived: Variable is derived. Information retrieval for derived variables always looks in the default tables (usually, the custom tables) if @@ -1534,7 +1535,7 @@ def get_variable( :class:`~esmvalcore.dataset.Dataset`, e.g. ``"Omon"`` for CMIP6 or ``"ocean"`` for CMIP7. short_name: - Variable's short name, e.g. "tos" for sea surface temperature. + Variable's short name, e.g. ``"tos"`` for sea surface temperature. branding_suffix: A suffix that will be appended to ``short_name`` when looking up the variable in the CMOR table, e.g. a @@ -1606,7 +1607,7 @@ def get_variable( :class:`~esmvalcore.dataset.Dataset`, e.g. ``"Omon"`` for CMIP6 or ``"ocean"`` for CMIP7. short_name: - Variable's short name, e.g. "tos" for sea surface temperature. + Variable's short name, e.g. ``"tos"`` for sea surface temperature. branding_suffix: A suffix that will be appended to ``short_name`` when looking up the variable in the CMOR table, e.g. a From 739fe733466927acfadfa0daf7a18361b06b2fe5 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Thu, 5 Feb 2026 13:13:56 +0100 Subject: [PATCH 35/47] Better error messages --- esmvalcore/cmor/table.py | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/esmvalcore/cmor/table.py b/esmvalcore/cmor/table.py index b6e6edbb1e..49138b6da5 100644 --- a/esmvalcore/cmor/table.py +++ b/esmvalcore/cmor/table.py @@ -19,7 +19,7 @@ import yaml -from esmvalcore.exceptions import RecipeError +from esmvalcore.exceptions import InvalidConfigParameter, RecipeError if TYPE_CHECKING: from collections.abc import Iterable @@ -358,15 +358,41 @@ def get_tables( raise ValueError(msg) cache_key = str(kwargs) if cache_key not in _TABLE_CACHE: + import_error_messsage = ( + "Please check your configuration. " + f"Current configuration is:\n\n{ + yaml.safe_dump( + {'projects': {project: {'cmor_table': kwargs}}}, + ) + }\n" + "Typical good values for 'type' are " + "'esmvalcore.cmor.table.CMIP6Info' for CMIP6-style CMOR tables or " + "'esmvalcore.cmor.table.CMIP5Info' for CMIP5-style CMOR tables." + ) module_name, cls_name = kwargs.pop("type").rsplit(".", 1) - module = importlib.import_module(module_name) - cls = getattr(module, cls_name) + try: + module = importlib.import_module(module_name) + except ModuleNotFoundError as exc: + msg = ( + f"Failed to import module '{module_name}' for CMOR table of " + f"project '{project}'. {import_error_messsage}" + ) + raise InvalidConfigParameter(msg) from exc + try: + cls = getattr(module, cls_name) + except AttributeError as exc: + msg = ( + f"Class '{cls_name}' for reading CMOR table of project " + f"'{project}' does not exist in module '{module_name}'. " + f"{import_error_messsage}" + ) + raise InvalidConfigParameter(msg) from exc tables = cls(**kwargs) if not isinstance(tables, InfoBase): msg = ( "`type` should be a subclass `esmvalcore.cmor.table.InfoBase`, " f"but your configuration for project '{project}' contains " - f"'{tables}' of type: '{type(tables)}'." + f"'{tables}' of type: '{type(tables)}'. {import_error_messsage}" ) raise TypeError(msg) _TABLE_CACHE[cache_key] = tables From 5a1852a62a0cc3a735c203e4609e59f2d6376d15 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Thu, 5 Feb 2026 13:23:58 +0100 Subject: [PATCH 36/47] Add tests --- .../integration/cmor/test_read_cmor_tables.py | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tests/integration/cmor/test_read_cmor_tables.py b/tests/integration/cmor/test_read_cmor_tables.py index caee63dd5d..f81b8cd5a7 100644 --- a/tests/integration/cmor/test_read_cmor_tables.py +++ b/tests/integration/cmor/test_read_cmor_tables.py @@ -16,6 +16,7 @@ read_cmor_tables, ) from esmvalcore.cmor.table import __file__ as root +from esmvalcore.exceptions import InvalidConfigParameter if TYPE_CHECKING: from esmvalcore.config import Session @@ -149,6 +150,43 @@ def test_get_tables_no_type( get_tables(session, "test") +def test_get_tables_non_existent_table_module( + session: Session, +) -> None: + session["projects"]["test"] = { + "cmor_table": { + "type": "tests.integration.cmor.does_not_exist.DoesNotExist", + }, + } + with pytest.raises( + InvalidConfigParameter, + match=( + r"Failed to import module 'tests.integration.cmor.does_not_exist' " + r"for CMOR table of project 'test'. Please check your configuration. " + ), + ): + get_tables(session, "test") + + +def test_get_tables_non_existent_table_class( + session: Session, +) -> None: + session["projects"]["test"] = { + "cmor_table": { + "type": "tests.integration.cmor.test_read_cmor_tables.NonExistentTable", + }, + } + with pytest.raises( + InvalidConfigParameter, + match=( + r"Class 'NonExistentTable' for reading CMOR table of project 'test' " + r"does not exist in module 'tests.integration.cmor.test_read_cmor_tables'. " + r"Please check your configuration." + ), + ): + get_tables(session, "test") + + class InvalidTable: pass From 8c0cbbdbdbd3933eb6e7b8b7e74fdeb1b6b5edf4 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Thu, 5 Feb 2026 17:21:42 +0100 Subject: [PATCH 37/47] Separate custom tables and reduce race conditions in tests --- esmvalcore/cmor/_fixes/native6/era5.py | 6 ++- esmvalcore/cmor/_utils.py | 6 ++- esmvalcore/cmor/table.py | 10 ++-- esmvalcore/cmor/tables/cmip3-custom/custom | 45 ++++++++++++++++ .../cmor/tables/cordex-custom/coordinates | 43 +++++++++++++++ esmvalcore/cmor/tables/{ => obs4mips}/VERSION | 0 .../CMOR_coordinates.dat | 0 .../tables/old-custom-coordinates/README.md | 5 ++ esmvalcore/config/_config_validators.py | 13 +++-- .../configurations/defaults/cmor_tables.yml | 3 +- esmvalcore/preprocessor/_derive/toz.py | 6 ++- esmvalcore/preprocessor/_regrid.py | 8 +-- .../cmor/_fixes/native6/test_era5.py | 11 ++-- .../integration/cmor/test_read_cmor_tables.py | 52 ++++++++----------- tests/integration/cmor/test_table.py | 35 ++++++------- tests/integration/io/test_local.py | 5 ++ tests/unit/config/test_config.py | 2 + tests/unit/config/test_data_sources.py | 2 + tests/unit/io/local/test_get_data_sources.py | 3 ++ tests/unit/preprocessor/test_configuration.py | 2 + 20 files changed, 184 insertions(+), 73 deletions(-) create mode 100644 esmvalcore/cmor/tables/cmip3-custom/custom create mode 100644 esmvalcore/cmor/tables/cordex-custom/coordinates rename esmvalcore/cmor/tables/{ => obs4mips}/VERSION (100%) rename esmvalcore/cmor/tables/{cmip5-custom => old-custom-coordinates}/CMOR_coordinates.dat (100%) create mode 100644 esmvalcore/cmor/tables/old-custom-coordinates/README.md diff --git a/esmvalcore/cmor/_fixes/native6/era5.py b/esmvalcore/cmor/_fixes/native6/era5.py index 26f2bc9e66..d1617a41ec 100644 --- a/esmvalcore/cmor/_fixes/native6/era5.py +++ b/esmvalcore/cmor/_fixes/native6/era5.py @@ -7,9 +7,9 @@ from iris.cube import CubeList from iris.util import reverse +import esmvalcore.cmor.table from esmvalcore.cmor._fixes.fix import Fix from esmvalcore.cmor._fixes.shared import add_scalar_height_coord -from esmvalcore.cmor.table import CMOR_TABLES from esmvalcore.iris_helpers import ( date2num, has_unstructured_grid, @@ -535,7 +535,9 @@ def _fix_coordinates( # noqa: C901 # (https://github.com/ESMValGroup/ESMValCore/issues/1029) if axis == "" and coord_def.name == "alevel": axis = "Z" - coord_def = CMOR_TABLES["CMIP6"].coords["plev19"] # noqa: PLW2901 + coord_def = esmvalcore.cmor.table.CMOR_TABLES["CMIP6"].coords[ # noqa: PLW2901 + "plev19" + ] coord = cube.coord(axis=axis) if axis == "T": coord.convert_units("days since 1850-1-1 00:00:00.0") diff --git a/esmvalcore/cmor/_utils.py b/esmvalcore/cmor/_utils.py index 2272a43e38..4429911419 100644 --- a/esmvalcore/cmor/_utils.py +++ b/esmvalcore/cmor/_utils.py @@ -5,7 +5,7 @@ import logging from typing import TYPE_CHECKING -from esmvalcore.cmor.table import CMOR_TABLES +import esmvalcore.cmor.table if TYPE_CHECKING: from collections.abc import Sequence @@ -68,7 +68,9 @@ def _get_alternative_generic_lev_coord( # Check if any of the allowed alternative coordinates is present in the # cube for allowed_alternative in allowed_alternatives: - cmor_coord = CMOR_TABLES[cmor_table_type].coords[allowed_alternative] + cmor_coord = esmvalcore.cmor.table.CMOR_TABLES[cmor_table_type].coords[ + allowed_alternative + ] if cube.coords(var_name=cmor_coord.out_name): cube_coord = cube.coord(var_name=cmor_coord.out_name) return (cmor_coord, cube_coord) diff --git a/esmvalcore/cmor/table.py b/esmvalcore/cmor/table.py index 49138b6da5..d29062c1a7 100644 --- a/esmvalcore/cmor/table.py +++ b/esmvalcore/cmor/table.py @@ -223,9 +223,8 @@ def read_cmor_tables(cfg_developer: Path | None = None) -> None: msg = "cfg_developer is not a Path-like object, got " raise TypeError(msg, cfg_developer) mtime = cfg_developer.stat().st_mtime - cmor_tables = _read_cmor_tables(cfg_developer, mtime) - CMOR_TABLES.clear() - CMOR_TABLES.update(cmor_tables) + global CMOR_TABLES # noqa: PLW0603 + CMOR_TABLES = _read_cmor_tables(cfg_developer, mtime) @lru_cache @@ -1502,7 +1501,10 @@ def __init__(self, cmor_tables_path: str | Path | None = None) -> None: self.tables[table.name] = table # First, read default custom tables from repository - self.paths = (Path(self._get_cmor_path("cmip5-custom")),) + self.paths = ( + Path(self._get_cmor_path("old-custom-coordinates")), + Path(self._get_cmor_path("cmip5-custom")), + ) # Second, if given, update default tables with user-defined custom # tables diff --git a/esmvalcore/cmor/tables/cmip3-custom/custom b/esmvalcore/cmor/tables/cmip3-custom/custom new file mode 100644 index 0000000000..d0b6cf54bc --- /dev/null +++ b/esmvalcore/cmor/tables/cmip3-custom/custom @@ -0,0 +1,45 @@ +table_id: Table custom + +!============ +variable_entry: lwcre +!============ +modeling_realm: atmos +!---------------------------------- +! Variable attributes: +!---------------------------------- +standard_name: +units: W m-2 +cell_methods: time: mean +cell_measures: area: areacella +long_name: TOA Longwave Cloud Radiative Effect +comment: at the top of the atmosphere (to be compared with satellite measurements) +!---------------------------------- +! Additional variable information: +!---------------------------------- +dimensions: longitude latitude time +type: real +positive: down +!---------------------------------- +! + +!============ +variable_entry: swcre +!============ +modeling_realm: atmos +!---------------------------------- +! Variable attributes: +!---------------------------------- +standard_name: +units: W m-2 +cell_methods: time: mean +cell_measures: area: areacella +long_name: TOA Shortwave Cloud Radiative Effect +comment: at the top of the atmosphere (to be compared with satellite measurements) +!---------------------------------- +! Additional variable information: +!---------------------------------- +dimensions: longitude latitude time +type: real +positive: down +!---------------------------------- +! diff --git a/esmvalcore/cmor/tables/cordex-custom/coordinates b/esmvalcore/cmor/tables/cordex-custom/coordinates new file mode 100644 index 0000000000..b2140ede8a --- /dev/null +++ b/esmvalcore/cmor/tables/cordex-custom/coordinates @@ -0,0 +1,43 @@ +table_id: Table custom + +!============ +axis_entry: plevs +!============ +!---------------------------------- +! Axis attributes: +!---------------------------------- +standard_name: air_pressure +units: Pa +axis: Z ! X, Y, Z, T (default: undeclared) +positive: down +long_name: pressure +!---------------------------------- +! Additional axis information: +!---------------------------------- +out_name: plev +valid_min: 0.0 +valid_max: 110000.0 +stored_direction: decreasing +type: double +must_have_bounds: no +!---------------------------------- +! + +!============ +axis_entry: tau +!============ +!---------------------------------- +! Axis attributes: +!---------------------------------- +standard_name: atmosphere_optical_thickness_due_to_cloud +units: 1 +long_name: cloud optical thickness +!---------------------------------- +! Additional axis information: +!---------------------------------- +out_name: tau +stored_direction: increasing +type: double +must_have_bounds: yes +!---------------------------------- +! diff --git a/esmvalcore/cmor/tables/VERSION b/esmvalcore/cmor/tables/obs4mips/VERSION similarity index 100% rename from esmvalcore/cmor/tables/VERSION rename to esmvalcore/cmor/tables/obs4mips/VERSION diff --git a/esmvalcore/cmor/tables/cmip5-custom/CMOR_coordinates.dat b/esmvalcore/cmor/tables/old-custom-coordinates/CMOR_coordinates.dat similarity index 100% rename from esmvalcore/cmor/tables/cmip5-custom/CMOR_coordinates.dat rename to esmvalcore/cmor/tables/old-custom-coordinates/CMOR_coordinates.dat diff --git a/esmvalcore/cmor/tables/old-custom-coordinates/README.md b/esmvalcore/cmor/tables/old-custom-coordinates/README.md new file mode 100644 index 0000000000..251fce6248 --- /dev/null +++ b/esmvalcore/cmor/tables/old-custom-coordinates/README.md @@ -0,0 +1,5 @@ +These are custom coordinate variables that were used with the custom CMIP5 +CMOR tables. They are problematic because they overwrite the standard CMIP5 +coordinates and thereby modify the CMIP5 CMOR table when used together with +that table. They are kept here until config-developer.yml and the associated +`esmvalcore.cmor.table.CustomInfo` class is retired in v2.16.0. diff --git a/esmvalcore/config/_config_validators.py b/esmvalcore/config/_config_validators.py index e2b9da448c..653f979a23 100644 --- a/esmvalcore/config/_config_validators.py +++ b/esmvalcore/config/_config_validators.py @@ -387,14 +387,13 @@ def validate_cmor_tables(value: dict) -> None: # preferably be avoided. This would require passing around session objects # instead of relying on global state (e.g. esmvalcore.config.CFG, # esmvalcore.cmor.tables.CMOR_TABLES). - esmvalcore.cmor.table.CMOR_TABLES.clear() - for project in value: - esmvalcore.cmor.table.CMOR_TABLES[project] = ( - esmvalcore.cmor.table.get_tables( - session={"projects": value}, # type: ignore[arg-type] - project=project, - ) + esmvalcore.cmor.table.CMOR_TABLES = { + project: esmvalcore.cmor.table.get_tables( + session={"projects": value}, # type: ignore[arg-type] + project=project, ) + for project in value + } def validate_projects( diff --git a/esmvalcore/config/configurations/defaults/cmor_tables.yml b/esmvalcore/config/configurations/defaults/cmor_tables.yml index 5da64dbf6c..61462f0761 100644 --- a/esmvalcore/config/configurations/defaults/cmor_tables.yml +++ b/esmvalcore/config/configurations/defaults/cmor_tables.yml @@ -24,12 +24,13 @@ projects: type: esmvalcore.cmor.table.CMIP3Info paths: - cmip3/Tables - - cmip5-custom + - cmip3-custom CORDEX: cmor_table: type: esmvalcore.cmor.table.CMIP5Info paths: - cordex/Tables + - cordex-custom - cmip5-custom obs4MIPs: cmor_table: diff --git a/esmvalcore/preprocessor/_derive/toz.py b/esmvalcore/preprocessor/_derive/toz.py index bf123c2a39..3f44f42684 100644 --- a/esmvalcore/preprocessor/_derive/toz.py +++ b/esmvalcore/preprocessor/_derive/toz.py @@ -4,7 +4,7 @@ import iris from scipy import constants -from esmvalcore.cmor.table import CMOR_TABLES +import esmvalcore.cmor.table from esmvalcore.iris_helpers import ignore_iris_vague_metadata_warnings from esmvalcore.preprocessor._regrid import extract_levels, regrid @@ -48,7 +48,9 @@ def add_longitude_coord(cube): def interpolate_hybrid_plevs(cube): """Interpolate hybrid pressure levels.""" # Use CMIP6's plev19 target levels (in Pa) - target_levels = CMOR_TABLES["CMIP6"].coords["plev19"].requested + target_levels = ( + esmvalcore.cmor.table.CMOR_TABLES["CMIP6"].coords["plev19"].requested + ) cube.coord("air_pressure").convert_units("Pa") return extract_levels( cube, diff --git a/esmvalcore/preprocessor/_regrid.py b/esmvalcore/preprocessor/_regrid.py index 486b831e79..461e75002a 100644 --- a/esmvalcore/preprocessor/_regrid.py +++ b/esmvalcore/preprocessor/_regrid.py @@ -28,11 +28,11 @@ from iris.cube import Cube from iris.util import broadcast_to_shape +import esmvalcore.cmor.table from esmvalcore.cmor._fixes.shared import ( add_altitude_from_plev, add_plev_from_altitude, ) -from esmvalcore.cmor.table import CMOR_TABLES from esmvalcore.iris_helpers import has_irregular_grid, has_unstructured_grid from esmvalcore.preprocessor._shared import ( _rechunk_aux_factory_dependencies, @@ -1403,15 +1403,15 @@ def get_cmor_levels(cmor_table: str, coordinate: str) -> list[float]: If the CMOR table is not defined, the coordinate does not specify any levels or the string is badly formatted. """ - if cmor_table not in CMOR_TABLES: + if cmor_table not in esmvalcore.cmor.table.CMOR_TABLES: msg = f"Level definition cmor_table '{cmor_table}' not available" raise ValueError(msg) - if coordinate not in CMOR_TABLES[cmor_table].coords: + if coordinate not in esmvalcore.cmor.table.CMOR_TABLES[cmor_table].coords: msg = f"Coordinate {coordinate} not available for {cmor_table}" raise ValueError(msg) - cmor = CMOR_TABLES[cmor_table].coords[coordinate] + cmor = esmvalcore.cmor.table.CMOR_TABLES[cmor_table].coords[coordinate] if cmor.requested: return [float(level) for level in cmor.requested] diff --git a/tests/integration/cmor/_fixes/native6/test_era5.py b/tests/integration/cmor/_fixes/native6/test_era5.py index cb7333fd7c..c899857bce 100644 --- a/tests/integration/cmor/_fixes/native6/test_era5.py +++ b/tests/integration/cmor/_fixes/native6/test_era5.py @@ -17,7 +17,8 @@ fix_accumulated_units, ) from esmvalcore.cmor.fix import fix_metadata -from esmvalcore.cmor.table import CMOR_TABLES, get_var_info +from esmvalcore.cmor.table import get_tables +from esmvalcore.config import CFG from esmvalcore.dataset import Dataset from esmvalcore.preprocessor import cmor_check_metadata @@ -26,18 +27,22 @@ f"{datetime.datetime.now().year}" ) +CMOR_TABLES = { + "native6": get_tables(CFG, "native6"), +} + def test_get_evspsbl_fix(): """Test whether the right fixes are gathered for a single variable.""" fix = Fix.get_fixes("native6", "ERA5", "E1hr", "evspsbl") - vardef = get_var_info("native6", "E1hr", "evspsbl") + vardef = CMOR_TABLES["native6"].get_variable("E1hr", "evspsbl") assert fix == [Evspsbl(vardef), AllVars(vardef), GenericFix(vardef)] def test_get_zg_fix(): """Test whether the right fix gets found again, for zg as well.""" fix = Fix.get_fixes("native6", "ERA5", "Amon", "zg") - vardef = get_var_info("native6", "E1hr", "evspsbl") + vardef = CMOR_TABLES["native6"].get_variable("E1hr", "evspsbl") assert fix == [Zg(vardef), AllVars(vardef), GenericFix(vardef)] diff --git a/tests/integration/cmor/test_read_cmor_tables.py b/tests/integration/cmor/test_read_cmor_tables.py index f81b8cd5a7..311db85da5 100644 --- a/tests/integration/cmor/test_read_cmor_tables.py +++ b/tests/integration/cmor/test_read_cmor_tables.py @@ -7,9 +7,9 @@ import pytest import yaml +import esmvalcore.cmor.table from esmvalcore.cmor.table import ( _TABLE_CACHE, - CMOR_TABLES, VariableInfo, clear_table_cache, get_tables, @@ -30,40 +30,30 @@ def test_read_cmor_tables_raiser(): assert "cow" in str(exc) -def test_read_cmor_tables(): +def test_read_cmor_tables_from_config_developer(monkeypatch): """Test that the function `read_cmor_tables` loads the tables correctly.""" + monkeypatch.setattr(esmvalcore.cmor.table, "CMOR_TABLES", {}) + read_cmor_tables() table_path = Path(root).parent / "tables" for project in "CMIP5", "CMIP6": - table = CMOR_TABLES[project] - assert table.paths == ( - table_path / project.lower() / "Tables", - table_path / f"{project.lower()}-custom", - ) + table = esmvalcore.cmor.table.CMOR_TABLES[project] + assert table.paths == (table_path / project.lower() / "Tables",) assert table.strict is True project = "OBS" - table = CMOR_TABLES[project] - assert table.paths == ( - table_path / "cmip5" / "Tables", - table_path / "cmip5-custom", - ) + table = esmvalcore.cmor.table.CMOR_TABLES[project] + assert table.paths == (table_path / "cmip5" / "Tables",) assert table.strict is False project = "OBS6" - table = CMOR_TABLES[project] - assert table.paths == ( - table_path / "cmip6" / "Tables", - table_path / "cmip6-custom", - ) + table = esmvalcore.cmor.table.CMOR_TABLES[project] + assert table.paths == (table_path / "cmip6" / "Tables",) assert table.strict is False project = "obs4MIPs" - table = CMOR_TABLES[project] - assert table.paths == ( - table_path / "obs4mips" / "Tables", - table_path / "cmip6-custom", - ) + table = esmvalcore.cmor.table.CMOR_TABLES[project] + assert table.paths == (table_path / "obs4mips" / "Tables",) assert table.strict is False @@ -83,7 +73,7 @@ def test_read_cmor_tables(): ("CMIP5", "Amon", "tas", None), ("CMIP5", "Amon", "alb", None), # custom derived variable ("CMIP3", "A1", "tas", None), - ("CMIP3", "A1", "alb", None), # custom derived variable + ("CMIP3", "A1", "lwcre", None), # custom derived variable ("CORDEX", "mon", "tas", None), ("CORDEX", "mon", "alb", None), # custom derived variable ("obs4MIPs", "Amon", "tas", None), @@ -119,7 +109,7 @@ def test_get_tables( mip, short_name, branding_suffix=branding_suffix, - derived=short_name == "alb", + derived=short_name in ("alb", "lwcre"), ) assert isinstance(vardef, VariableInfo) assert vardef.short_name @@ -290,8 +280,9 @@ def test_clear_table_cache(session: Session) -> None: ) -def test_read_custom_cmor_tables_config_developer(tmp_path): +def test_read_custom_cmor_tables_config_developer(tmp_path, monkeypatch): """Test reading of custom CMOR tables.""" + monkeypatch.setattr(esmvalcore.cmor.table, "CMOR_TABLES", {}) (tmp_path / "CMOR_newvarfortesting.dat").write_text(CMOR_NEWVAR_ENTRY) (tmp_path / "CMOR_netcre.dat").write_text(CMOR_NETCRE_ENTRY) (tmp_path / "CMOR_coordinates.dat").write_text(CMOR_NEWCOORD_ENTRY) @@ -312,12 +303,13 @@ def test_read_custom_cmor_tables_config_developer(tmp_path): read_cmor_tables(cfg_file) - assert len(CMOR_TABLES) == 2 - assert "CMIP6" in CMOR_TABLES - assert "custom" in CMOR_TABLES + assert len(esmvalcore.cmor.table.CMOR_TABLES) == 2 + assert "CMIP6" in esmvalcore.cmor.table.CMOR_TABLES + assert "custom" in esmvalcore.cmor.table.CMOR_TABLES - custom_table = CMOR_TABLES["custom"] + custom_table = esmvalcore.cmor.table.CMOR_TABLES["custom"] assert custom_table.paths == ( + Path(root).parent / "tables" / "old-custom-coordinates", Path(root).parent / "tables" / "cmip5-custom", tmp_path, ) @@ -334,7 +326,7 @@ def test_read_custom_cmor_tables_config_developer(tmp_path): assert netcre.units == "K" assert netcre.long_name == "This is New" - cmip6_table = CMOR_TABLES["CMIP6"] + cmip6_table = esmvalcore.cmor.table.CMOR_TABLES["CMIP6"] assert cmip6_table.default is custom_table # Restore default tables diff --git a/tests/integration/cmor/test_table.py b/tests/integration/cmor/test_table.py index 21e68f75d1..3e7f43f455 100644 --- a/tests/integration/cmor/test_table.py +++ b/tests/integration/cmor/test_table.py @@ -423,7 +423,7 @@ def variables_info(self) -> CMIP3Info: return CMIP3Info( paths=[ Path("cmip3/Tables"), - Path("cmip5-custom"), + Path("cmip3-custom"), ], strict=True, ) @@ -490,11 +490,11 @@ def test_aermon_ta_succes_if_strict(self, variables_info): assert var.short_name == "ta" assert var.frequency == "" - def test_omon_toz_succes_if_strict(self, variables_info): - """Get toz does not fail with Omon if not strict.""" + def test_derived_variable_success_if_strict(self, variables_info): + """Get swcre does not fail with Omon if not strict.""" variables_info.strict = False - var = variables_info.get_variable("O1", "toz") - assert var.short_name == "toz" + var = variables_info.get_variable("O1", "swcre") + assert var.short_name == "swcre" assert var.frequency == "" @@ -506,6 +506,7 @@ def variables_info(self) -> CMIP5Info: return CMIP5Info( paths=[ Path("cordex/Tables"), + Path("cordex-custom"), Path("cmip5-custom"), ], ) @@ -536,6 +537,7 @@ def variables_info(self) -> CustomInfo: def test_repr(self, variables_info: CustomInfo) -> None: builtin_tables_path = Path(esmvalcore.cmor.__file__).parent / "tables" expected_paths = [ + builtin_tables_path / "old-custom-coordinates", builtin_tables_path / "cmip5-custom", ] result = repr(variables_info) @@ -544,32 +546,29 @@ def test_repr(self, variables_info: CustomInfo) -> None: def test_custom_tables_default_location(self, variables_info): """Test constructor with default tables location.""" custom_info = CustomInfo() - expected_cmor_folder = os.path.join( - os.path.dirname(esmvalcore.cmor.__file__), - "tables", - "cmip5-custom", + builtin_tables_path = Path(esmvalcore.cmor.__file__).parent / "tables" + default_paths = ( + builtin_tables_path / "old-custom-coordinates", + builtin_tables_path / "cmip5-custom", ) - assert custom_info.paths == (Path(expected_cmor_folder),) + assert custom_info.paths == default_paths assert custom_info.tables["custom"] assert custom_info.coords def test_custom_tables_location(self, variables_info): """Test constructor with custom tables location.""" cmor_path = os.path.dirname(os.path.realpath(esmvalcore.cmor.__file__)) - default_cmor_tables_path = os.path.join( - cmor_path, - "tables", - "cmip5-custom", - ) cmor_tables_path = os.path.join(cmor_path, "tables", "cmip5") cmor_tables_path = os.path.abspath(cmor_tables_path) custom_info = CustomInfo(cmor_tables_path) - assert custom_info.paths == ( - Path(default_cmor_tables_path), - Path(cmor_tables_path), + builtin_tables_path = Path(esmvalcore.cmor.__file__).parent / "tables" + default_paths = ( + builtin_tables_path / "old-custom-coordinates", + builtin_tables_path / "cmip5-custom", ) + assert custom_info.paths == (*default_paths, Path(cmor_tables_path)) assert custom_info.tables["custom"] assert custom_info.coords diff --git a/tests/integration/io/test_local.py b/tests/integration/io/test_local.py index 6818bf85ac..0e52ba1901 100644 --- a/tests/integration/io/test_local.py +++ b/tests/integration/io/test_local.py @@ -10,6 +10,7 @@ import yaml import esmvalcore +import esmvalcore.cmor.table from esmvalcore.config import CFG from esmvalcore.io.local import ( LocalDataSource, @@ -69,6 +70,7 @@ def create_tree(path, filenames=None, symlinks=None): @pytest.mark.parametrize("cfg", CONFIG["get_output_file"]) def test_get_output_file(monkeypatch, cfg): """Test getting output name for preprocessed files.""" + monkeypatch.setattr(esmvalcore.cmor.table, "CMOR_TABLES", {}) monkeypatch.setitem( CFG, "config_developer_file", @@ -101,6 +103,7 @@ def test_find_files(monkeypatch, root, cfg): pprint.pformat(cfg["variable"]), ) project = cfg["variable"]["project"] + monkeypatch.setattr(esmvalcore.cmor.table, "CMOR_TABLES", {}) monkeypatch.setitem( CFG, "config_developer_file", @@ -132,6 +135,7 @@ def test_find_files_with_facets(monkeypatch, root): break project = cfg["variable"]["project"] + monkeypatch.setattr(esmvalcore.cmor.table, "CMOR_TABLES", {}) monkeypatch.setitem( CFG, "config_developer_file", @@ -193,6 +197,7 @@ def test_find_data(root, cfg): def test_select_invalid_drs_structure(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(esmvalcore.cmor.table, "CMOR_TABLES", {}) monkeypatch.setitem( CFG, "config_developer_file", diff --git a/tests/unit/config/test_config.py b/tests/unit/config/test_config.py index 9a7c9b8775..20ec067ab4 100644 --- a/tests/unit/config/test_config.py +++ b/tests/unit/config/test_config.py @@ -6,6 +6,7 @@ import pytest import yaml +import esmvalcore.cmor.table import esmvalcore.config._config from esmvalcore.cmor.check import CheckLevels from esmvalcore.config import CFG, _config, _config_validators @@ -322,6 +323,7 @@ def test_get_ignored_warnings_none(project, step): def test_get_ignored_warnings_emac(monkeypatch: pytest.MonkeyPatch) -> None: """Test ``get_ignored_warnings``.""" + monkeypatch.setattr(esmvalcore.cmor.table, "CMOR_TABLES", {}) monkeypatch.setitem( CFG, "config_developer_file", diff --git a/tests/unit/config/test_data_sources.py b/tests/unit/config/test_data_sources.py index fce0b3a625..1f80a69614 100644 --- a/tests/unit/config/test_data_sources.py +++ b/tests/unit/config/test_data_sources.py @@ -5,6 +5,7 @@ import pytest +import esmvalcore.cmor.table import esmvalcore.config._data_sources import esmvalcore.local from esmvalcore.exceptions import InvalidConfigParameter @@ -38,6 +39,7 @@ def test_load_legacy_data_sources( session["projects"][project].pop("data", None) session["search_esgf"] = search_esgf session["download_dir"] = "~/climate_data" + monkeypatch.setattr(esmvalcore.cmor.table, "CMOR_TABLES", {}) monkeypatch.setitem( esmvalcore.local.CFG, "config_developer_file", diff --git a/tests/unit/io/local/test_get_data_sources.py b/tests/unit/io/local/test_get_data_sources.py index c086947a57..d3ccf59e71 100644 --- a/tests/unit/io/local/test_get_data_sources.py +++ b/tests/unit/io/local/test_get_data_sources.py @@ -6,6 +6,7 @@ import pytest import esmvalcore +import esmvalcore.cmor.table from esmvalcore.config import CFG from esmvalcore.io.local import LocalDataSource from esmvalcore.local import DataSource, _get_data_sources @@ -33,6 +34,7 @@ ) def test_get_data_sources(monkeypatch, rootpath_drs): # Make sure that default config-developer file is used + monkeypatch.setattr(esmvalcore.cmor.table, "CMOR_TABLES", {}) monkeypatch.setitem( CFG, "config_developer_file", @@ -52,6 +54,7 @@ def test_get_data_sources(monkeypatch, rootpath_drs): def test_get_data_sources_nodefault(monkeypatch): # Make sure that default config-developer file is used + monkeypatch.setattr(esmvalcore.cmor.table, "CMOR_TABLES", {}) monkeypatch.setitem( CFG, "config_developer_file", diff --git a/tests/unit/preprocessor/test_configuration.py b/tests/unit/preprocessor/test_configuration.py index 8b79c149ab..b89ae9ad73 100644 --- a/tests/unit/preprocessor/test_configuration.py +++ b/tests/unit/preprocessor/test_configuration.py @@ -8,6 +8,7 @@ import pytest import esmvalcore +import esmvalcore.cmor.table from esmvalcore.config import CFG from esmvalcore.dataset import Dataset from esmvalcore.exceptions import RecipeError @@ -154,6 +155,7 @@ def test_get_preprocessor_filename_falls_back_to_config_developer( session: Session, ) -> None: """Test the function `_get_preprocessor_filename`.""" + monkeypatch.setattr(esmvalcore.cmor.table, "CMOR_TABLES", {}) monkeypatch.setitem( CFG, "config_developer_file", From b3de2a8c485ab9641bd48042a7eb484db8619ee9 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Fri, 6 Feb 2026 11:13:24 +0100 Subject: [PATCH 38/47] Restore custom tables test from config-developer and add another patch of CMOR tables when testing config-developer.yml --- esmvalcore/cmor/_utils.py | 2 +- tests/integration/cmor/test_read_cmor_tables.py | 9 +++++++++ tests/unit/config/test_config_validator.py | 4 +++- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/esmvalcore/cmor/_utils.py b/esmvalcore/cmor/_utils.py index 4429911419..9728f8bcad 100644 --- a/esmvalcore/cmor/_utils.py +++ b/esmvalcore/cmor/_utils.py @@ -165,7 +165,7 @@ def _get_new_generic_level_coord( New generic level coordinate. """ - new_coord = generic_level_coord.generic_lev_coords[new_coord_name] # type: ignore[index] # Is this a bug? + new_coord = generic_level_coord.generic_lev_coords[new_coord_name] new_coord.generic_level = True new_coord.generic_lev_coords = var_info.coordinates[ generic_level_coord_name diff --git a/tests/integration/cmor/test_read_cmor_tables.py b/tests/integration/cmor/test_read_cmor_tables.py index 311db85da5..e05f7c6637 100644 --- a/tests/integration/cmor/test_read_cmor_tables.py +++ b/tests/integration/cmor/test_read_cmor_tables.py @@ -56,6 +56,15 @@ def test_read_cmor_tables_from_config_developer(monkeypatch): assert table.paths == (table_path / "obs4mips" / "Tables",) assert table.strict is False + project = "custom" + table = esmvalcore.cmor.table.CMOR_TABLES[project] + assert table.paths == ( + table_path / "old-custom-coordinates", + table_path / "cmip5-custom", + ) + assert table.coords + assert table.tables["custom"] + @pytest.mark.parametrize( ( diff --git a/tests/unit/config/test_config_validator.py b/tests/unit/config/test_config_validator.py index 06fbac5936..8d3e594f02 100644 --- a/tests/unit/config/test_config_validator.py +++ b/tests/unit/config/test_config_validator.py @@ -5,6 +5,7 @@ import yaml import esmvalcore +import esmvalcore.cmor.table from esmvalcore import __version__ as current_version from esmvalcore.config import CFG from esmvalcore.config._config_validators import ( @@ -330,8 +331,9 @@ def test_validate_config_developer_none(): assert path is None -def test_validate_config_developer(tmp_path): +def test_validate_config_developer(tmp_path, monkeypatch): """Test ``validate_config_developer``.""" + monkeypatch.setattr(esmvalcore.cmor.table, "CMOR_TABLES", {}) custom_table_path = ( Path(esmvalcore.__file__).parent / "cmor" / "tables" / "cmip5-custom" ) From e67aea3ef08f537d89164e3afd9081bff2447bca Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Fri, 6 Feb 2026 11:30:49 +0100 Subject: [PATCH 39/47] Add another deprecation warning --- esmvalcore/_main.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/esmvalcore/_main.py b/esmvalcore/_main.py index 5e5668865b..5c4446bdb1 100644 --- a/esmvalcore/_main.py +++ b/esmvalcore/_main.py @@ -33,6 +33,7 @@ import os import re import sys +import warnings from importlib.metadata import entry_points from pathlib import Path from typing import TYPE_CHECKING @@ -371,8 +372,6 @@ def get_config_user( version 2.16.0. Use the ``copy`` method instead. """ - import warnings - from esmvalcore.exceptions import ESMValCoreDeprecationWarning deprecation_msg = ( @@ -417,6 +416,20 @@ def get_config_developer( If not provided, the file will be copied to `~/.esmvaltool`. """ + from esmvalcore.exceptions import ESMValCoreDeprecationWarning + + deprecation_msg = ( + "The config-developer.yml file and the associated " + "'esmvaltool config get_config_developer' command are deprecated " + "and support for them will be removed in ESMValCore version 2.16.0. " + "Please configure data sources, cmor tables, and preprocessor " + "filename templates under `projects` instead." + ) + warnings.warn( + deprecation_msg, + category=ESMValCoreDeprecationWarning, + stacklevel=1, + ) in_file = Path(__file__).parent / "config-developer.yml" if path is None: out_file = Path.home() / ".esmvaltool" / "config-developer.yml" From 1dec95ffcd0f84dd2700b2b5ab19d918630dc04e Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Fri, 6 Feb 2026 13:52:51 +0100 Subject: [PATCH 40/47] Ensure config_developer_file populates esmvalcore.cmor.table.CMOR_TABLES for backward compatibility --- esmvalcore/config/_validated_config.py | 40 ++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/esmvalcore/config/_validated_config.py b/esmvalcore/config/_validated_config.py index 505434c419..0369d0d836 100644 --- a/esmvalcore/config/_validated_config.py +++ b/esmvalcore/config/_validated_config.py @@ -4,7 +4,7 @@ import pprint import warnings -from collections.abc import MutableMapping +from collections.abc import Mapping, MutableMapping from contextlib import contextmanager from copy import deepcopy from typing import TYPE_CHECKING, Any, ClassVar @@ -17,7 +17,7 @@ from ._config_validators import ValidationError if TYPE_CHECKING: - from collections.abc import Callable, Generator, Mapping + from collections.abc import Callable, Generator # The code for this class was take from matplotlib (v3.3) and modified to @@ -81,6 +81,42 @@ def __setitem__(self, key, val): self._mapping[key] = cval + def update(self, *args, **kwargs): # noqa: C901 + """Update configuration with key/value pairs from mapping or kwargs.""" + # This ensures the config_developer_file is always set last when doing + # an update, so the esmvalcore.cmor.table.CMOR_TABLES variable + # is populated with the values from config_developer_file, which is + # needed for backward compatibility. + # TODO: remove in v2.16.0. + config_developer_file_not_set = object() + config_developer_file = config_developer_file_not_set + if args: + if len(args) > 1: + msg = ( + f"Expected at most 1 positional argument, got {len(args)}" + ) + raise TypeError(msg) + other = args[0] + if isinstance(other, Mapping): + for key in other: + if key == "config_developer_file": + config_developer_file = other[key] + continue + self[key] = other[key] + else: + for key, value in other: + if key == "config_developer_file": + config_developer_file = other[key] + continue + self[key] = value + for key, value in kwargs.items(): + if key == "config_developer_file": + config_developer_file = value + continue + self[key] = value + if config_developer_file is not config_developer_file_not_set: + self["config_developer_file"] = config_developer_file + def __getitem__(self, key): """Return value mapped by key.""" try: From 610c23da46d0e108212881e1ea510f9e33e39721 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Fri, 6 Feb 2026 15:18:37 +0100 Subject: [PATCH 41/47] Fix support for legacy data sources without custom config-developer.yml --- esmvalcore/_main.py | 3 +++ esmvalcore/config/_config.py | 8 ++++++-- esmvalcore/config/_config_validators.py | 1 + esmvalcore/local.py | 22 ++++++++++++++++++++-- 4 files changed, 30 insertions(+), 4 deletions(-) diff --git a/esmvalcore/_main.py b/esmvalcore/_main.py index 5c4446bdb1..cff2c3c993 100644 --- a/esmvalcore/_main.py +++ b/esmvalcore/_main.py @@ -788,6 +788,9 @@ def display(lines, out): raise except SuppressedError as exc: # Hide the stack trace for RecipeErrors + if not logger.handlers: + # Add a logging handler if main failed to do so. + logging.basicConfig() logger.error("%s", exc) logger.debug("Stack trace for debugging:", exc_info=True) sys.exit(1) diff --git a/esmvalcore/config/_config.py b/esmvalcore/config/_config.py index aaf0ad47c8..3da50569be 100644 --- a/esmvalcore/config/_config.py +++ b/esmvalcore/config/_config.py @@ -84,7 +84,10 @@ def warn_if_old_extra_facets_exist() -> None: ) -def load_config_developer(cfg_file: Path) -> dict: +def load_config_developer( + cfg_file: Path, + set_cmor_tables: bool = True, +) -> dict: """Read the developer's configuration file.""" with open(cfg_file, encoding="utf-8") as file: cfg = yaml.safe_load(file) @@ -112,7 +115,8 @@ def load_config_developer(cfg_file: Path) -> dict: settings["input_dir"][site] = normalized_drs CFG[project] = settings - read_cmor_tables(cfg_file) + if set_cmor_tables: + read_cmor_tables(cfg_file) return cfg diff --git a/esmvalcore/config/_config_validators.py b/esmvalcore/config/_config_validators.py index 653f979a23..d2a39c8bc8 100644 --- a/esmvalcore/config/_config_validators.py +++ b/esmvalcore/config/_config_validators.py @@ -683,4 +683,5 @@ def deprecate_config_developer_file( # https://github.com/ESMValGroup/ESMValCore/pull/2213 _deprecated_options_defaults: dict[str, Any] = { "extra_facets_dir": [], # TODO: remove in v2.15.0 + "download_dir": "~/climate_data", # TODO: remove in v2.16.0 } diff --git a/esmvalcore/local.py b/esmvalcore/local.py index 850d32d0ef..97c3822412 100644 --- a/esmvalcore/local.py +++ b/esmvalcore/local.py @@ -15,7 +15,12 @@ from typing import TYPE_CHECKING, Any from esmvalcore.config import CFG -from esmvalcore.config._config import get_ignored_warnings, get_project_config +from esmvalcore.config._config import CFG as CONFIG_DEVELOPER +from esmvalcore.config._config import ( + get_ignored_warnings, + get_project_config, + load_config_developer, +) from esmvalcore.io.local import ( LocalDataSource, LocalFile, @@ -37,6 +42,17 @@ logger = logging.getLogger(__name__) +def _ensure_config_developer_drs() -> Path: + """Ensure that directory structure from config-developer.yml is loaded.""" + config_developer_file = CFG.get("config_developer_file") + if not config_developer_file: + config_developer_file = Path(__file__).parent / "config-developer.yml" + if not CONFIG_DEVELOPER: + # Load the config-developer.yml file, but do not update the CMOR tables. + load_config_developer(config_developer_file, set_cmor_tables=False) + return config_developer_file + + def _select_drs(input_type: str, project: str, structure: str) -> list[str]: """Select the directory structure of input path.""" cfg = get_project_config(project) @@ -61,6 +77,7 @@ def _select_drs(input_type: str, project: str, structure: str) -> list[str]: def _get_data_sources(project: str) -> list[LocalDataSource]: """Get a list of data sources.""" + config_developer_file = _ensure_config_developer_drs() rootpaths = CFG["rootpath"] default_drs = { "CMIP3": "ESGF", @@ -110,7 +127,7 @@ def _get_data_sources(project: str) -> list[LocalDataSource]: "and 'drs' settings and the path templates from '%s'" ), project, - CFG["config_developer_file"], + config_developer_file, ) _LEGACY_DATA_SOURCES_WARNED.add(project) return sources @@ -264,6 +281,7 @@ def find_files( def _get_output_file(variable: dict[str, Any], preproc_dir: Path) -> Path: """Return the full path to the output (preprocessed) file.""" + _ensure_config_developer_drs() project = variable["project"] cfg = get_project_config(project) if project not in _GET_OUTPUT_FILE_WARNED: From b5064fd3555664a558d7586d7e9e1ef31adbe308 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Fri, 6 Feb 2026 17:27:00 +0100 Subject: [PATCH 42/47] Improve test coverage and fix bug --- esmvalcore/config/_validated_config.py | 2 +- tests/integration/io/test_local.py | 22 ++++++++++++- tests/unit/config/test_config_object.py | 43 +++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 2 deletions(-) diff --git a/esmvalcore/config/_validated_config.py b/esmvalcore/config/_validated_config.py index 0369d0d836..c7124b718a 100644 --- a/esmvalcore/config/_validated_config.py +++ b/esmvalcore/config/_validated_config.py @@ -106,7 +106,7 @@ def update(self, *args, **kwargs): # noqa: C901 else: for key, value in other: if key == "config_developer_file": - config_developer_file = other[key] + config_developer_file = value continue self[key] = value for key, value in kwargs.items(): diff --git a/tests/integration/io/test_local.py b/tests/integration/io/test_local.py index 0e52ba1901..71b114f35f 100644 --- a/tests/integration/io/test_local.py +++ b/tests/integration/io/test_local.py @@ -11,6 +11,8 @@ import esmvalcore import esmvalcore.cmor.table +import esmvalcore.config._config +import esmvalcore.local from esmvalcore.config import CFG from esmvalcore.io.local import ( LocalDataSource, @@ -81,6 +83,22 @@ def test_get_output_file(monkeypatch, cfg): assert output_file == expected +@pytest.mark.parametrize("cfg", CONFIG["get_output_file"]) +def test_get_output_file_no_config_developer(monkeypatch, cfg): + """Test getting output name for preprocessed files.""" + monkeypatch.setattr(esmvalcore.local, "CFG", {}) + monkeypatch.setattr(esmvalcore.cmor.table, "CMOR_TABLES", {}) + monkeypatch.setattr(esmvalcore.config._config, "CFG", {}) + monkeypatch.setattr(esmvalcore.local, "CONFIG_DEVELOPER", {}) + output_file = _get_output_file(cfg["variable"], cfg["preproc_dir"]) + expected = Path(cfg["output_file"]) + assert output_file == expected + # This test ensures that only the directory structure bits of + # config-developer.yml have been loaded. + assert esmvalcore.config._config.CFG + assert not esmvalcore.cmor.table.CMOR_TABLES + + @pytest.fixture def root(tmp_path): """Root function for tests.""" @@ -91,7 +109,7 @@ def root(tmp_path): @pytest.mark.parametrize("cfg", CONFIG["get_input_filelist"]) -def test_find_files(monkeypatch, root, cfg): +def test_find_files(monkeypatch, root, cfg, mocker): """Test retrieving input filelist.""" if "drs" not in cfg: pytest.skip( @@ -103,6 +121,7 @@ def test_find_files(monkeypatch, root, cfg): pprint.pformat(cfg["variable"]), ) project = cfg["variable"]["project"] + mocker.patch.object(esmvalcore.local, "_ensure_config_developer_drs") monkeypatch.setattr(esmvalcore.cmor.table, "CMOR_TABLES", {}) monkeypatch.setitem( CFG, @@ -126,6 +145,7 @@ def test_find_files(monkeypatch, root, cfg): ] assert [Path(f) for f in input_filelist] == sorted(ref_files) assert [Path(g) for g in globs] == sorted(ref_globs) + esmvalcore.local._ensure_config_developer_drs.assert_called_once() def test_find_files_with_facets(monkeypatch, root): diff --git a/tests/unit/config/test_config_object.py b/tests/unit/config/test_config_object.py index 83f9b4e64e..3ecbe5db68 100644 --- a/tests/unit/config/test_config_object.py +++ b/tests/unit/config/test_config_object.py @@ -5,6 +5,7 @@ import pytest import esmvalcore +import esmvalcore.cmor.table import esmvalcore.config._config_object from esmvalcore.config import CFG, Config, Session from esmvalcore.config._config_object import DEFAULT_CONFIG_DIR @@ -44,6 +45,48 @@ def test_config_update(): config.update(fail_dict) +@pytest.mark.parametrize("update_format", ["mapping", "kwargs", "tuple"]) +def test_config_update_config_developer_set_last( + monkeypatch: pytest.MonkeyPatch, + update_format: str, +) -> None: + monkeypatch.setattr(esmvalcore.cmor.table, "CMOR_TABLES", {}) + new_config = { + "config_developer_file": Path(esmvalcore.__file__).parent + / "config-developer.yml", + "projects": { + "CMIP6": { + "cmor_table": { + "type": "esmvalcore.cmor.table.NoInfo", + }, + }, + }, + } + config = Config({"output_dir": "directory"}) + if update_format == "mapping": + config.update(new_config) + elif update_format == "kwargs": + config.update(**new_config) + elif update_format == "tuple": + config.update(tuple(new_config.items())) + + assert len(esmvalcore.cmor.table.CMOR_TABLES) > 1 + assert "CMIP6" in esmvalcore.cmor.table.CMOR_TABLES + assert isinstance( + esmvalcore.cmor.table.CMOR_TABLES["CMIP6"], + esmvalcore.cmor.table.CMIP6Info, + ) + + +def test_config_update_too_many_args() -> None: + config = Config({"output_dir": "directory"}) + with pytest.raises( + TypeError, + match=r"Expected at most 1 positional argument, got 2", + ): + config.update(1, 2) + + def test_set_bad_item(): config = Config({"output_dir": "config"}) with pytest.raises(InvalidConfigParameter) as err_exc: From 81317b13e2ab01006351a011dada7cd9d07fa847 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Mon, 9 Feb 2026 10:34:04 +0100 Subject: [PATCH 43/47] Update siextent units in CMIP6 custom table --- esmvalcore/cmor/tables/cmip6-custom/CMIP6_custom.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvalcore/cmor/tables/cmip6-custom/CMIP6_custom.json b/esmvalcore/cmor/tables/cmip6-custom/CMIP6_custom.json index 7628aa821e..d399838a75 100644 --- a/esmvalcore/cmor/tables/cmip6-custom/CMIP6_custom.json +++ b/esmvalcore/cmor/tables/cmip6-custom/CMIP6_custom.json @@ -1054,7 +1054,7 @@ "out_name": "siextent", "standard_name": "", "type": "real", - "units": "m2", + "units": "1", "valid_max": "", "valid_min": "" }, From a982de23cb651aebc0aa0b8145101fadcd8b1f09 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Mon, 9 Feb 2026 12:47:06 +0100 Subject: [PATCH 44/47] Improve docs with suggestions by @schlunma --- doc/develop/fixing_data.rst | 14 +++++--------- doc/quickstart/configure.rst | 9 +++------ 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/doc/develop/fixing_data.rst b/doc/develop/fixing_data.rst index 6f6504bd6f..1c3f73b0bf 100644 --- a/doc/develop/fixing_data.rst +++ b/doc/develop/fixing_data.rst @@ -364,10 +364,7 @@ below from the lowest level of strictness to the highest: CMORized data and may not work as expected with non-CMORized data. Our ambition is that :ref:`preprocessor functions ` support - data that follows the - `CF Conventions `_, while - :ref:`diagnostics ` are only expected to work with - CMORized data. + data that follows the `CF Conventions `_. .. _add_new_fix_native_datasets: @@ -383,7 +380,7 @@ under project ``native6``. CMOR Table Configuration ------------------------ -An example of a CMOR table configuration for projects used for native datasets is given in +An example of a CMOR table configuration for projects used for native datasets is given here: .. literalinclude:: ../configurations/defaults/cmor_tables.yml :language: yaml @@ -392,9 +389,9 @@ An example of a CMOR table configuration for projects used for native datasets i :start-at: # Observational and reanalysis data that can be read in its native format by ESMValCore. :end-before: CESM: -The option ``strict: false`` is convenient for these projects, if you +The option ``strict: false`` is convenient for these projects if you want to make use of the feature that looks in all tables instead of -only the one specified by the ``mip`` facet in the recipe or +only the one specified by the ``mip`` facet in the :ref:`recipe ` or :class:`~esmvalcore.dataset.Dataset`: like this, a custom variable only needs to be defined for a single ``mip`` and can then be used with all ``mip`` s. @@ -414,8 +411,7 @@ To allow ESMValCore to locate the data files, use the following steps: :caption: ``native6`` standard directory organization in ``data-local-esmvaltool.yml`` :end-before: # Data that has been CMORized by ESMValTool according to the CMIP6 standard. - this is preferred. This is usually the case for native reanalysis/observational - datasets. + this is preferred. - If moving the data into a particular directory structure is not possible, the ``data`` entry of the ``native6`` project could be complemented diff --git a/doc/quickstart/configure.rst b/doc/quickstart/configure.rst index 7068564005..fdd695b463 100644 --- a/doc/quickstart/configure.rst +++ b/doc/quickstart/configure.rst @@ -696,7 +696,7 @@ The following project-specific options are available: * - ``cmor_table`` - :ref:`CMOR tables ` are used to define the variables that ESMValCore can work with. Refer to :ref:`cmor_table_configuration` for available options. - :obj:`dict` - - ``{}`` + - ``{"type": "esmvalcore.cmor.table.NoInfo"}`` * - ``data`` - Data sources are used to find input data and have to be configured before running the tool. Refer to :ref:`config-data-sources` for details. - :obj:`dict` @@ -753,10 +753,7 @@ for loading and saving the data in the :ref:`recipe ` or CMORized data and may not work as expected with non-CMORized data. Our ambition is that :ref:`preprocessor functions ` support - data that follows the - `CF Conventions `_, while - :ref:`diagnostics ` are only expected to work with - CMORized data. + data that follows the `CF Conventions `_. .. _config-data-sources: @@ -1170,7 +1167,7 @@ Upgrade instructions for finding files -------------------------------------- The ``input_dir``, ``input_file``, and ``ignore_warnings`` settings have -been replaced by the :class:`esmvalcore.io.local.LocalDataSource`, which can be +been replaced by :class:`esmvalcore.io.local.LocalDataSource`, which can be configured via :ref:`data sources `. Example 1: A config-developer.yml file specifying a directory structure for From 5a2147dfd88d9638ef09248a4acdfa6bf0e0d2b0 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Mon, 9 Feb 2026 12:52:56 +0100 Subject: [PATCH 45/47] Remove impractical suggestion on how to organize data --- doc/develop/fixing_data.rst | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/doc/develop/fixing_data.rst b/doc/develop/fixing_data.rst index 1c3f73b0bf..1859ef41d1 100644 --- a/doc/develop/fixing_data.rst +++ b/doc/develop/fixing_data.rst @@ -413,34 +413,6 @@ To allow ESMValCore to locate the data files, use the following steps: this is preferred. -- If moving the data into a particular directory structure is not possible, - the ``data`` entry of the ``native6`` project could be complemented - with another data source that goes under a new - key representing the data organization (such as ``MY_DATA_ORG``), and - these sub-entries can use an arbitrary list of ``{placeholders}``. - - Example: - - .. code-block:: yaml - - projects: - native6: - data: - MY_DATA_ORG: - type: esmvalcore.io.local.LocalDataSource - rootpath: /path/to/data - dirname_template: "{dataset}/{exp}/{simulation}/{version}/{type}" - filename_template: '{simulation}_*.nc' - - would allow the tool to find your native data (e.g., a ``dataset`` called ``MYDATA``) - that is for example located in ``/path/to/data/MYDATA/amip/run1/42-0/atm/run1_1979.nc`` - if you use the following dataset entry in your recipe - - .. code-block:: yaml - - datasets: - - {project: native6, dataset: MYDATA, exp: amip, simulation: run1, version: 42-0, type: atm} - - If you want to use a dedicated project for your native dataset (this is usually the case for native model output): From 2ee9aad4311f987a5cf1558d3bd26f0d95289995 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Mon, 9 Feb 2026 13:03:13 +0100 Subject: [PATCH 46/47] Make defaults/cmor_tables.yml a link --- doc/quickstart/configure.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/quickstart/configure.rst b/doc/quickstart/configure.rst index fdd695b463..b9c46f159b 100644 --- a/doc/quickstart/configure.rst +++ b/doc/quickstart/configure.rst @@ -718,7 +718,8 @@ CMOR table configuration :ref:`CMOR tables ` are used to define the variables that ESMValCore can work with. -Default values are provided in ``defaults/cmor_tables.yml``, +Default values are provided in +`defaults/cmor_tables.yml `__, for example: .. literalinclude:: ../configurations/defaults/cmor_tables.yml From b43a466ca4a30bd8e0bb5e5b6d4c78893ff26a9a Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Wed, 11 Feb 2026 12:49:06 +0100 Subject: [PATCH 47/47] Add note about which files are read --- esmvalcore/cmor/table.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/esmvalcore/cmor/table.py b/esmvalcore/cmor/table.py index d29062c1a7..f7fbbbb555 100644 --- a/esmvalcore/cmor/table.py +++ b/esmvalcore/cmor/table.py @@ -680,7 +680,9 @@ class CMIP6Info(InfoBase): tables in the `esmvalcore/cmor/tables `_ directory, or any other path. The built-in tables will be used if the - path is relative and exists in the built-in tables directory. + path is relative and exists in the built-in tables directory. Only files + with the extension ``.json`` in the specified paths will be read as a + CMOR tables, any other files will be ignored. """ @@ -1245,7 +1247,8 @@ class CMIP5Info(InfoBase): tables in the `esmvalcore/cmor/tables `_ directory, or any other path. The built-in tables will be used if the - path is relative and exists in the built-in tables directory. + path is relative and exists in the built-in tables directory. Any file + in the specified paths will be read as a CMOR table. """ @@ -1448,7 +1451,8 @@ class CMIP3Info(CMIP5Info): tables in the `esmvalcore/cmor/tables `_ directory, or any other path. The built-in tables will be used if the - path is relative and exists in the built-in tables directory. + path is relative and exists in the built-in tables directory. Any file + in the specified paths will be read as a CMOR table. """