diff --git a/.github/instructions/*.instructions.md b/.github/instructions/*.instructions.md new file mode 120000 index 0000000000..fc66fcc90c --- /dev/null +++ b/.github/instructions/*.instructions.md @@ -0,0 +1 @@ +doc/contributing.rst \ No newline at end of file diff --git a/.gitignore b/.gitignore index ee821184de..1b9d52962e 100644 --- a/.gitignore +++ b/.gitignore @@ -109,3 +109,7 @@ doc/_sidebar.rst.inc # ESMF log files *.ESMF_LogFile + + +#Ignore vscode AI rules +.github/instructions/codacy.instructions.md diff --git a/doc/changelog.rst b/doc/changelog.rst index 4a2af7ba42..e6ba93f230 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -813,7 +813,7 @@ Bug fixes ~~~~~~~~~ - Respect ``ignore_warnings`` settings from the project configuration in config-developer.yml in :func:`esmvalcore.dataset.Dataset.load` (:pull:`2046`) by :user:`schlunma` -- Fixed usage of custom location for :ref:`custom CMOR tables ` (:pull:`2052`) by :user:`schlunma` +- Fixed usage of custom location for custom CMOR tables (:pull:`2052`) by :user:`schlunma` - Fix issue with writing index.html when :ref:`running a recipe ` with ``--resume-from`` (:pull:`2055`) by :user:`bouweandela` - Fixed bug in ICON CMORizer that lead to shifted time coordinates (:pull:`2038`) by :user:`schlunma` - Include ``-`` in allowed characters for bibtex references (:pull:`2097`) by :user:`alistairsellar` diff --git a/doc/contributing.rst b/doc/contributing.rst index 3c5a3e8ca6..cc5a33956c 100644 --- a/doc/contributing.rst +++ b/doc/contributing.rst @@ -72,7 +72,9 @@ Please keep the following considerations in mind when programming: what exactly is being changed. - :ref:`preprocessor_functions` are Python functions (and not classes) so they are easy to understand and implement for scientific contributors. -- No additional CMOR checks should be implemented inside preprocessor functions. +- No additional CMOR checks should be implemented inside preprocessor functions, + though it is our ambition to make preprocessor functions work with any data that + follows the `CF Conventions `_. The input cube is fixed and confirmed to follow the specification in `esmvalcore/cmor/tables `__ before applying any other preprocessor functions. @@ -82,7 +84,8 @@ Please keep the following considerations in mind when programming: recipe to the relevant CMOR table. - The ESMValCore package is based on :ref:`iris `. Preprocessor functions should preferably be small and just call the relevant - iris code. + iris code. This automatically ensures that the preprocessor functions work with + data that follows the CF Conventions. Code that is more involved and more broadly applicable than just in the ESMValCore, should be implemented in iris instead. - Any settings in the recipe that can be checked before loading the data should diff --git a/doc/develop/fixing_data.rst b/doc/develop/fixing_data.rst index 119fef5f09..1859ef41d1 100644 --- a/doc/develop/fixing_data.rst +++ b/doc/develop/fixing_data.rst @@ -316,6 +316,16 @@ missing coordinate you can create a fix for this model: Customizing checker strictness ============================== +The baseline case for ESMValCore input data is fully +:ref:`CMOR compliant ` data and this is checked when data is +loaded by :meth:`esmvalcore.dataset.Dataset.load`. + +However, it is possible to disable these checks completely by +:ref:`configuring the project ` so it uses the +:class:`esmvalcore.cmor.table.NoInfo` CMOR table, +or to adjust the strictness of the checks using the :ref:`configuration option +` ``check_level``. + The data checker classifies its issues using four different levels of severity. From highest to lowest: @@ -345,6 +355,16 @@ below from the lowest level of strictness to the highest: strictness. Mostly useful for checking datasets that you have produced, to be sure that future users will not be distracted by inoffensive warnings. +.. warning:: + + While it is possible to work with datasets that are not described in a CMOR + table or only partially follow the CMOR standards, the + :ref:`preprocessor functions ` and + :ref:`diagnostics ` have been designed to work with + CMORized data and may not work as expected with non-CMORized data. + + Our ambition is that :ref:`preprocessor functions ` support + data that follows the `CF Conventions `_. .. _add_new_fix_native_datasets: @@ -357,14 +377,23 @@ under project ``native6``. .. _add_new_fix_native_datasets_config: -Configuration -------------- +CMOR Table Configuration +------------------------ -An example of a configuration in ``config-developer.yml`` for projects used for -native datasets is given :ref:`here `. -Make sure to use the option ``cmor_strict: false`` for these projects if you -want to make use of :ref:`custom_cmor_tables`. -This allows reading arbitrary variables from native datasets. +An example of a CMOR table configuration for projects used for native datasets is given here: + +.. literalinclude:: ../configurations/defaults/cmor_tables.yml + :language: yaml + :caption: Example native data format projects in ``defaults/cmor_tables.yml`` + :prepend: projects: + :start-at: # Observational and reanalysis data that can be read in its native format by ESMValCore. + :end-before: CESM: + +The option ``strict: false`` is convenient for these projects if you +want to make use of the feature that looks in all tables instead of +only the one specified by the ``mip`` facet in the :ref:`recipe ` or +:class:`~esmvalcore.dataset.Dataset`: like this, a custom variable only needs to +be defined for a single ``mip`` and can then be used with all ``mip`` s. .. _add_new_fix_native_datasets_locate_data: @@ -373,89 +402,56 @@ Locate data To allow ESMValCore to locate the data files, use the following steps: - - If you want to use the ``native6`` project (recommended for datasets whose - input files can be easily moved to the usual ``native6`` directory - structure given by the :ref:`configuration option ` - ``rootpath``; this is usually the case for native reanalysis/observational - datasets): - - The entry ``native6`` of ``config-developer.yml`` should be complemented - with sub-entries for ``input_dir`` and ``input_file`` that go under a new - key representing the data organization (such as ``MY_DATA_ORG``), and - these sub-entries can use an arbitrary list of ``{placeholders}``. - Example : - - .. code-block:: yaml - - native6: - ... - input_dir: - default: 'Tier{tier}/{dataset}/{version}/{frequency}/{short_name}' - MY_DATA_ORG: '{dataset}/{exp}/{simulation}/{version}/{type}' - input_file: - default: '*.nc' - MY_DATA_ORG: '{simulation}_*.nc' - ... - - To find your native data (e.g., called ``MYDATA``) that is for example - located in ``{rootpath}/MYDATA/amip/run1/42-0/atm/run1_1979.nc`` - (``{rootpath}`` is ESMValTool's ``rootpath`` :ref:`configuration option - ` for the project ``native6``), use the following dataset - entry in your recipe - - .. code-block:: yaml - - datasets: - - {project: native6, dataset: MYDATA, exp: amip, simulation: run1, version: 42-0, type: atm} - - and make sure to use the following :ref:`configuration option - ` ``drs``: - - .. code-block:: yaml - - drs: - native6: MY_DATA_ORG - - - If you want to use a dedicated project for your native dataset - (recommended for datasets for which you cannot control the location of the - input files; this is usually the case for native model output): - - A new entry for the project needs to be added to ``config-developer.yml``. - For example, for the ICON model, create a new project ``ICON``: - - .. code-block:: yaml - - ICON: - ... - input_dir: - default: - - '{exp}' - - '{exp}/outdata' - - '{exp}/output' - input_file: - default: '{exp}_{var_type}*.nc' - ... - - To find your ICON data that is for example located in files like - ``{rootpath}/amip/amip_atm_2d_ml_20000101T000000Z.nc`` (``{rootpath}`` is - ESMValCore's :ref:`configuration option ` ``rootpath`` for - the project ``ICON``), use the following dataset entry in your recipe: - - .. code-block:: yaml - - datasets: - - {project: ICON, dataset: ICON, exp: amip} - - Please note the duplication of the name ``ICON`` in ``project`` and - ``dataset``, which is necessary to comply with ESMValTool's data finding - and CMORizing functionalities. - For other native models, ``dataset`` could also refer to a subversion of - the model. - Note that it is possible to predefine facets via :ref:`extra facets - `. - In this ICON example, the facet ``var_type`` is :download:`predefined - ` - for many variables. +- If you want to use the ``native6`` project, recommended for datasets whose + input files can be easily moved to the usual ``native6`` directory + structure given by + + .. literalinclude:: ../configurations/data-local-esmvaltool.yml + :language: yaml + :caption: ``native6`` standard directory organization in ``data-local-esmvaltool.yml`` + :end-before: # Data that has been CMORized by ESMValTool according to the CMIP6 standard. + + this is preferred. + +- If you want to use a dedicated project for your native dataset + (this is usually the case for native model output): + + A new entry for the project needs to be added under :ref:`config-projects`. + For example, for the ICON model, create a new project ``ICON`` and define + its data sources: + + .. literalinclude:: ../configurations/data-native-icon.yml + :language: yaml + :caption: ``ICON`` standard directory organization in ``data-native-icon.yml`` + + and a CMOR table configuration: + + .. literalinclude:: ../configurations/defaults/cmor_tables.yml + :language: yaml + :caption: ``ICON`` CMOR table configuration from ``defaults/cmor_tables.yml`` + :prepend: projects: + :start-at: ICON: + :end-at: strict: false + + To find your ICON data that is, for example, located in files like + ``~/climate_data/amip/amip_atm_2d_ml_20000101T000000Z.nc``, use the following + dataset entry in your recipe: + + .. code-block:: yaml + + datasets: + - {project: ICON, dataset: ICON, exp: amip} + + Please note the duplication of the name ``ICON`` in ``project`` and + ``dataset``, which is necessary to comply with ESMValTool's data finding + and CMORizing functionalities. + For other native models, ``dataset`` could also refer to a subversion of + the model. + Note that it is possible to predefine facets via :ref:`extra facets + `. + In this ICON example, the facet ``var_type`` is :download:`predefined + ` + for many variables. .. _add_new_fix_native_datasets_fix_data: diff --git a/doc/quickstart/configure.rst b/doc/quickstart/configure.rst index 4e39b94d94..b9c46f159b 100644 --- a/doc/quickstart/configure.rst +++ b/doc/quickstart/configure.rst @@ -693,6 +693,10 @@ The following project-specific options are available: - Description - Type - Default value + * - ``cmor_table`` + - :ref:`CMOR tables ` are used to define the variables that ESMValCore can work with. Refer to :ref:`cmor_table_configuration` for available options. + - :obj:`dict` + - ``{"type": "esmvalcore.cmor.table.NoInfo"}`` * - ``data`` - Data sources are used to find input data and have to be configured before running the tool. Refer to :ref:`config-data-sources` for details. - :obj:`dict` @@ -706,6 +710,52 @@ The following project-specific options are available: - :obj:`str` - Refer to :ref:`config-preprocessor-filename-template`. +.. _cmor_table_configuration: + +CMOR table configuration +------------------------ + +:ref:`CMOR tables ` are used to define the variables that ESMValCore +can work with. + +Default values are provided in +`defaults/cmor_tables.yml `__, +for example: + +.. literalinclude:: ../configurations/defaults/cmor_tables.yml + :language: yaml + :caption: First few lines of ``defaults/cmor_tables.yml`` + :end-before: CMIP3: + +The ``type`` parameter defines which class is used to read the CMOR tables for a +project, it should be a subclass of :class:`esmvalcore.cmor.table.InfoBase`. +The other parameters are passed as keyword arguments to the class when it is +created. See :mod:`esmvalcore.cmor.table` for a description of the built-in +classes for reading CMOR tables and their parameters. + +Most users will not need to change the CMOR table configuration. However, if you +have variables you would like to use which are not in the standard tables, +it is recommended that you start from the default configuration and extend the +list of ``paths`` with the directory where your custom CMOR tables are stored. + +If you have data that is not described in a CMOR table at all and do not need +interoperability with typical climate model data, you can use +the :class:`esmvalcore.cmor.table.NoInfo` class to indicate that no CMOR table +is available. In that case you will need to provide all necessary facets +for loading and saving the data in the :ref:`recipe ` or +:class:`~esmvalcore.dataset.Dataset`, and +:ref:`CMOR checks ` will be skipped. + +.. warning:: + + While it is possible to work with datasets that are not described in a CMOR + table, the :ref:`preprocessor functions ` and + :ref:`diagnostics ` have been designed to work with + CMORized data and may not work as expected with non-CMORized data. + + Our ambition is that :ref:`preprocessor functions ` support + data that follows the `CF Conventions `_. + .. _config-data-sources: Data sources @@ -1098,179 +1148,162 @@ resort. Developer configuration file ============================ -Most users and diagnostic developers will not need to change this file, -but it may be useful to understand its content. -The settings from this file are being moved to the -:ref:`new configuration system `. In particular, the -``input_dir``, ``input_file``, and ``ignore_warnings`` settings have already -been replaced by the :class:`esmvalcore.io.local.LocalDataSource` that can be +.. deprecated:: 2.14.0 + + The developer configuration file is deprecated and will no longer be supported + in v2.16.0. Please use the :ref:`project-specific configuration ` + instead. + + See the `v2.13.0 `__ + documentation for previous usage of the developer configuration file. + +.. warning:: + + Make sure that **no** ``config-developer.yml`` file is saved + in the ESMValCore configuration directories (see + :ref:`config_overview` for details), as it does not contain configuration + options that are valid in the new configuration system. + +Upgrade instructions for finding files +-------------------------------------- + +The ``input_dir``, ``input_file``, and ``ignore_warnings`` settings have +been replaced by :class:`esmvalcore.io.local.LocalDataSource`, which can be configured via :ref:`data sources `. -The developer configuration file will be installed along with ESMValCore and can -also be viewed on GitHub: -`esmvalcore/config-developer.yml -`_. -This configuration file describes the CMOR tables for several -key projects (CMIP6, CMIP5, obs4MIPs, OBS6, OBS), and for native output data for some -models (ICON, IPSL, ... see :ref:`configure_native_models`). -Users can get a copy of this file with default values by running +Example 1: A config-developer.yml file specifying a directory structure for +CMIP6 data: -.. code-block:: bash +.. code:: yaml - esmvaltool config get_config_developer --path=${TARGET_FOLDER} + CMIP6: + input_dir: + ESGF: "{project}/{activity}/{institute}/{dataset}/{exp}/{ensemble}/{mip}/{short_name}/{grid}/{version}" + input_file: "{short_name}_{mip}_{dataset}_{exp}_{ensemble}_{grid}*.nc" -If the option ``--path`` is omitted, the file will be created in -``~/.esmvaltool``. +and associated ``rootpath`` and ``drs`` settings: -.. note:: +.. code:: yaml - Remember to change the configuration option ``config_developer_file`` if you - want to use a custom config developer file. + rootpath: + CMIP6: ~/climate_data + drs: + CMIP6: ESGF -.. warning:: +would translate to the following new configuration: - For now, make sure that the custom ``config-developer.yml`` is **not** saved - in the ESMValTool/Core configuration directories (see - :ref:`config_yaml_files` for details). - This will change in the future due to the :ref:`redesign of ESMValTool/Core's - configuration `. +.. code:: yaml -Example of the CMIP6 project configuration: + projects: + CMIP6: + data: + local: + type: esmvalcore.io.local.LocalDataSource + rootpath: ~/climate_data + dirname_template: "{project}/{activity}/{institute}/{dataset}/{exp}/{ensemble}/{mip}/{short_name}/{grid}/{version}" + filename_template: "{short_name}_{mip}_{dataset}_{exp}_{ensemble}_{grid}*.nc" -.. code-block:: yaml +Upgrade instructions for naming preprocessor output files +--------------------------------------------------------- - CMIP6: - cmor_type: 'CMIP6' - cmor_strict: true +The ``output_file`` setting has been replaced by the +``preprocessor_filename_template`` settings described in +:ref:`config-preprocessor-filename-template`. -.. _cmor_table_configuration: +Example 1: A config-developer.yml file specifying preprocessor output filenames +for CMIP6 data: -Project CMOR table configuration --------------------------------- - -ESMValCore comes bundled with several :ref:`CMOR tables `, which are stored in the directory -`esmvalcore/cmor/tables `_. -These are copies of the tables available from `PCMDI `_. - -For every ``project`` that can be used in the recipe, there are four settings -related to CMOR table settings available: - -* ``cmor_type``: can be ``CMIP5`` if the CMOR table is in the same format as the - CMIP5 table or ``CMIP6`` if the table is in the same format as the CMIP6 table. -* ``cmor_strict``: if this is set to ``false``, the CMOR table will be - extended with variables from the :ref:`custom_cmor_tables` (by default loaded - from the ``esmvalcore/cmor/tables/custom`` directory) and it is possible to - use variables with a ``mip`` which is different from the MIP table in which - they are defined. Note that this option is always enabled for - :ref:`derived variables `. -* ``cmor_path``: path to the CMOR table. - Relative paths are with respect to `esmvalcore/cmor/tables`_. - Defaults to the value provided in ``cmor_type`` written in lower case. -* ``cmor_default_table_prefix``: Prefix that needs to be added to the ``mip`` - to get the name of the file containing the ``mip`` table. - Defaults to the value provided in ``cmor_type``. - -.. _custom_cmor_tables: - -Custom CMOR tables ------------------- +.. code:: yaml -As mentioned in the previous section, the CMOR tables of projects that use -``cmor_strict: false`` will be extended with custom CMOR tables. -For :ref:`derived variables ` (the ones with ``derive: -true`` in the recipe), the custom CMOR tables will always be considered. -By default, these custom tables are loaded from `esmvalcore/cmor/tables/custom -`_. -However, by using the special project ``custom`` in the -``config-developer.yml`` file with the option ``cmor_path``, a custom location -for these custom CMOR tables can be specified. -In this case, the default custom tables are extended with those entries from -the custom location (in case of duplication, the custom location tables take -precedence). + CMIP6: + output_file: "{project}_{dataset}_{mip}_{exp}_{ensemble}_{short_name}_{grid}" -Example: + +would translate to the following new configuration: + +.. code:: yaml + + projects: + CMIP6: + preprocessor_filename_template: "{project}_{dataset}_{mip}_{exp}_{ensemble}_{short_name}_{grid}" + +Upgrade instructions for using custom CMOR tables +------------------------------------------------- + +The CMOR tables can now be configured via :ref:`cmor_table_configuration`. The +following mapping applies: + +- ``cmor_type`` has been replaced by ``type`` +- ``cmor_strict`` has been replaced by ``strict`` +- ``cmor_path`` has been replaced by ``paths`` +- ``cmor_default_table_prefix`` is no longer needed. + +Because it is now possible to configure multiple paths to directories containing +CMOR tables per project, the ``custom`` project for specifying additional custom +CMOR tables is no longer needed nor supported. + +Example 1: A config-developer.yml file specifying different CMIP6 CMOR tables +than the default ones, augmented by the default custom CMOR tables: .. code-block:: yaml - custom: - cmor_path: ~/my/own/custom_tables - -This path can be given as relative path (relative to `esmvalcore/cmor/tables`_) -or as absolute path. -Other options given for this special table will be ignored. - -Custom tables in this directory need to follow the naming convention -``CMOR_{short_name}.dat`` and need to be given in CMIP5 format. - -Example for the file ``CMOR_asr.dat``: - -.. code-block:: - - SOURCE: CMIP5 - !============ - variable_entry: asr - !============ - modeling_realm: atmos - !---------------------------------- - ! Variable attributes: - !---------------------------------- - standard_name: - units: W m-2 - cell_methods: time: mean - cell_measures: area: areacella - long_name: Absorbed shortwave radiation - !---------------------------------- - ! Additional variable information: - !---------------------------------- - dimensions: longitude latitude time - type: real - positive: down - !---------------------------------- - ! - -It is also possible to use a special coordinates file ``CMOR_coordinates.dat``, -which will extend the entries from the default one -(`esmvalcore/cmor/tables/custom/CMOR_coordinates.dat -`_). - - -.. _configure_native_models: - -Configuring datasets in native format -------------------------------------- - -ESMValCore can be configured for handling native model output formats and -specific reanalysis/observation datasets without preliminary reformatting. -These datasets can be either hosted under the ``native6`` project (mostly -native reanalysis/observational datasets) or under a dedicated project, e.g., -``ICON`` (mostly native models). + CMIP6: + cmor_path: /path/to/cmip6-cmor-tables + cmor_strict: true + cmor_type: CMIP6 -Example: +would translate to the following new configuration: .. code-block:: yaml - native6: - cmor_strict: false - cmor_type: 'CMIP6' - cmor_default_table_prefix: 'CMIP6_' + projects: + CMIP6: + cmor_table: + type: esmvalcore.cmor.table.CMIP6Info + strict: true + paths: + - /path/to/cmip6-cmor-tables + - cmip6-custom - ICON: - cmor_strict: false - cmor_type: 'CMIP6' - cmor_default_table_prefix: 'CMIP6_' +where the ``cmip6-custom`` relative path refers to +`esmvalcore/cmor/tables/cmip6-custom `__ +and ``type`` refers to :class:`esmvalcore.cmor.table.CMIP6Info`. -A detailed description on how to add support for further native datasets is -given :ref:`here `. +Example 2: A config-developer.yml file specifying additional custom CMOR tables: -.. hint:: +.. code:: yaml + + CMIP6: + cmor_path: cmip6 + cmor_strict: true + cmor_type: CMIP6 + custom: + cmor_path: /path/to/custom-cmip5-style-tables + +would translate to the following new configuration: - When using native datasets, it might be helpful to specify a custom location - for the :ref:`custom_cmor_tables`. - This allows reading arbitrary variables from native datasets. - Note that this requires the option ``cmor_strict: false`` in the - :ref:`project configuration ` used for the native - model output. +.. code:: yaml + projects: + CMIP6: + cmor_table: + type: esmvalcore.cmor.table.CMIP6Info + strict: true + paths: + - cmip6 + - cmip6-custom + - /path/to/custom-cmip6-style-tables + +where the relative paths ``cmip6`` and ``cmip6-custom`` refer to +`esmvalcore/cmor/tables/cmip6 `__ +and +`esmvalcore/cmor/tables/cmip6-custom `__ +respectively and the directory ``/path/to/custom-cmip6-style-tables`` contains +the additional custom CMOR tables in CMIP6 format. A script to translate custom +CMIP5 format tables to CMIP6 format tables is available +`here `__. +Note that it is no longer possible to mix CMIP6-style tables with custom +CMIP5-style tables for the same project. .. _config-ref: diff --git a/doc/quickstart/find_data.rst b/doc/quickstart/find_data.rst index fa1a07705e..76cbd61e2e 100644 --- a/doc/quickstart/find_data.rst +++ b/doc/quickstart/find_data.rst @@ -85,15 +85,6 @@ project, e.g., ``ICON`` (mostly native models). A detailed description of how to include new native datasets is given :ref:`here `. -.. hint:: - - When using native datasets, it might be helpful to specify a custom location - for the :ref:`custom_cmor_tables`. - This allows reading arbitrary variables from native datasets. - Note that this requires the option ``cmor_strict: false`` in the - :ref:`project configuration ` used for the native - model output. - .. _read_native_obs: Supported native reanalysis/observational datasets diff --git a/doc/recipe/preprocessor.rst b/doc/recipe/preprocessor.rst index 78a1e9a598..7918f49bf8 100644 --- a/doc/recipe/preprocessor.rst +++ b/doc/recipe/preprocessor.rst @@ -2949,6 +2949,10 @@ recipe: target metadata is read. If not given, use the short names of the corresponding variables defined in the recipe. +* ``target_branding_suffix`` (:obj:`str`; optional): Variable branding suffix from which + target metadata is read. + If not given, use the branding suffixes of the corresponding variables defined in + the recipe. * ``strict`` (:obj:`str`; optional, default: ``True``): If ``True``, raise an error if desired metadata cannot be read for variable ``target_short_name`` of MIP table ``target_mip`` and project ``target_project``. diff --git a/doc/reference/cmor_tables.rst b/doc/reference/cmor_tables.rst index b969a938ab..20b15e3d55 100644 --- a/doc/reference/cmor_tables.rst +++ b/doc/reference/cmor_tables.rst @@ -49,10 +49,10 @@ by an underscore and the ``branding_suffix``. For example, the facets ``project: CMIP7, mip: atmos, short_name: tas, branding_suffix: tavg-h2m-hxy-u`` select one of the near-surface air temperature variables in the CMIP7 atmos table: -.. literalinclude:: ../../esmvalcore/cmor/tables/cmip7/Tables/CMIP7_atmos.json +.. literalinclude:: ../../esmvalcore/cmor/tables/cmip7/tables/CMIP7_atmos.json :start-at: "tas_tavg-h2m-hxy-u": { :end-at: }, - :caption: One of the ``tas`` variable definitions in the CMIP7 atmos table at `esmvalcore/cmor/tables/cmip7/Tables/CMIP7_atmos.json `__. + :caption: One of the ``tas`` variable definitions in the CMIP7 atmos table at `esmvalcore/cmor/tables/cmip7/tables/CMIP7_atmos.json `__. For other projects, the facet ``branding_suffix`` can also be used to distinguish between variables from the same CMOR table that share the same ``short_name``, diff --git a/esmvalcore/_main.py b/esmvalcore/_main.py index 5e5668865b..cff2c3c993 100644 --- a/esmvalcore/_main.py +++ b/esmvalcore/_main.py @@ -33,6 +33,7 @@ import os import re import sys +import warnings from importlib.metadata import entry_points from pathlib import Path from typing import TYPE_CHECKING @@ -371,8 +372,6 @@ def get_config_user( version 2.16.0. Use the ``copy`` method instead. """ - import warnings - from esmvalcore.exceptions import ESMValCoreDeprecationWarning deprecation_msg = ( @@ -417,6 +416,20 @@ def get_config_developer( If not provided, the file will be copied to `~/.esmvaltool`. """ + from esmvalcore.exceptions import ESMValCoreDeprecationWarning + + deprecation_msg = ( + "The config-developer.yml file and the associated " + "'esmvaltool config get_config_developer' command are deprecated " + "and support for them will be removed in ESMValCore version 2.16.0. " + "Please configure data sources, cmor tables, and preprocessor " + "filename templates under `projects` instead." + ) + warnings.warn( + deprecation_msg, + category=ESMValCoreDeprecationWarning, + stacklevel=1, + ) in_file = Path(__file__).parent / "config-developer.yml" if path is None: out_file = Path.home() / ".esmvaltool" / "config-developer.yml" @@ -775,6 +788,9 @@ def display(lines, out): raise except SuppressedError as exc: # Hide the stack trace for RecipeErrors + if not logger.handlers: + # Add a logging handler if main failed to do so. + logging.basicConfig() logger.error("%s", exc) logger.debug("Stack trace for debugging:", exc_info=True) sys.exit(1) diff --git a/esmvalcore/_recipe/check.py b/esmvalcore/_recipe/check.py index 1aae4b6aef..bb935cbc0a 100644 --- a/esmvalcore/_recipe/check.py +++ b/esmvalcore/_recipe/check.py @@ -48,6 +48,7 @@ def align_metadata(step_settings: dict[str, Any]) -> None: project = step_settings.get("target_project") mip = step_settings.get("target_mip") short_name = step_settings.get("target_short_name") + branding_suffix = step_settings.get("target_branding_suffix") strict = step_settings.get("strict", True) # Any missing arguments will be reported later @@ -55,7 +56,12 @@ def align_metadata(step_settings: dict[str, Any]) -> None: return try: - _get_var_info(project, mip, short_name) + _get_var_info( + project, + mip, + short_name, + branding_suffix=branding_suffix, + ) except ValueError as exc: if strict: msg = ( diff --git a/esmvalcore/cmor/_fixes/fix.py b/esmvalcore/cmor/_fixes/fix.py index 26c587f9ed..7f2bfc216e 100644 --- a/esmvalcore/cmor/_fixes/fix.py +++ b/esmvalcore/cmor/_fixes/fix.py @@ -572,7 +572,7 @@ def _fix_alternative_generic_level_coords(self, cube: Cube) -> Cube: # Make sure to update variable information with actual generic # level coordinate if one has been found; this is necessary for # subsequent fixes - if standard_name: + if name is not None: new_generic_level_coord = _get_new_generic_level_coord( self.vardef, cmor_coord, diff --git a/esmvalcore/cmor/_fixes/native6/era5.py b/esmvalcore/cmor/_fixes/native6/era5.py index 26f2bc9e66..d1617a41ec 100644 --- a/esmvalcore/cmor/_fixes/native6/era5.py +++ b/esmvalcore/cmor/_fixes/native6/era5.py @@ -7,9 +7,9 @@ from iris.cube import CubeList from iris.util import reverse +import esmvalcore.cmor.table from esmvalcore.cmor._fixes.fix import Fix from esmvalcore.cmor._fixes.shared import add_scalar_height_coord -from esmvalcore.cmor.table import CMOR_TABLES from esmvalcore.iris_helpers import ( date2num, has_unstructured_grid, @@ -535,7 +535,9 @@ def _fix_coordinates( # noqa: C901 # (https://github.com/ESMValGroup/ESMValCore/issues/1029) if axis == "" and coord_def.name == "alevel": axis = "Z" - coord_def = CMOR_TABLES["CMIP6"].coords["plev19"] # noqa: PLW2901 + coord_def = esmvalcore.cmor.table.CMOR_TABLES["CMIP6"].coords[ # noqa: PLW2901 + "plev19" + ] coord = cube.coord(axis=axis) if axis == "T": coord.convert_units("days since 1850-1-1 00:00:00.0") diff --git a/esmvalcore/cmor/_utils.py b/esmvalcore/cmor/_utils.py index da8eddd759..9728f8bcad 100644 --- a/esmvalcore/cmor/_utils.py +++ b/esmvalcore/cmor/_utils.py @@ -5,7 +5,7 @@ import logging from typing import TYPE_CHECKING -from esmvalcore.cmor.table import CMOR_TABLES +import esmvalcore.cmor.table if TYPE_CHECKING: from collections.abc import Sequence @@ -68,7 +68,9 @@ def _get_alternative_generic_lev_coord( # Check if any of the allowed alternative coordinates is present in the # cube for allowed_alternative in allowed_alternatives: - cmor_coord = CMOR_TABLES[cmor_table_type].coords[allowed_alternative] + cmor_coord = esmvalcore.cmor.table.CMOR_TABLES[cmor_table_type].coords[ + allowed_alternative + ] if cube.coords(var_name=cmor_coord.out_name): cube_coord = cube.coord(var_name=cmor_coord.out_name) return (cmor_coord, cube_coord) @@ -130,7 +132,7 @@ def _get_new_generic_level_coord( var_info: VariableInfo, generic_level_coord: CoordinateInfo, generic_level_coord_name: str, - new_coord_name: str | None, + new_coord_name: str, ) -> CoordinateInfo: """Get new generic level coordinate. diff --git a/esmvalcore/cmor/check.py b/esmvalcore/cmor/check.py index 3e27da363d..21121be515 100644 --- a/esmvalcore/cmor/check.py +++ b/esmvalcore/cmor/check.py @@ -425,14 +425,18 @@ def _check_dim_names(self): "exist", ) - def _check_generic_level_dim_names(self, key, coordinate): + def _check_generic_level_dim_names( + self, + key: str, + coordinate: CoordinateInfo, + ) -> None: """Check name of generic level coordinate.""" if coordinate.generic_lev_coords: - (standard_name, out_name, name) = _get_generic_lev_coord_names( + (_, out_name, name) = _get_generic_lev_coord_names( self._cube, coordinate, ) - if standard_name: + if name is not None: if not out_name: self.report_error( f"Generic level coordinate {key} has wrong var_name.", diff --git a/esmvalcore/cmor/table.py b/esmvalcore/cmor/table.py index 71c519497b..f7fbbbb555 100644 --- a/esmvalcore/cmor/table.py +++ b/esmvalcore/cmor/table.py @@ -7,26 +7,41 @@ from __future__ import annotations import copy -import errno import glob +import importlib import json import logging import os from collections import Counter from functools import lru_cache, total_ordering from pathlib import Path -from typing import Union +from typing import TYPE_CHECKING, Self import yaml -from esmvalcore.exceptions import RecipeError +from esmvalcore.exceptions import InvalidConfigParameter, RecipeError + +if TYPE_CHECKING: + from collections.abc import Iterable + from io import TextIOWrapper + + from esmvalcore.config import Config, Session + from esmvalcore.typing import Facets logger = logging.getLogger(__name__) -CMORTable = Union["CMIP3Info", "CMIP5Info", "CMIP6Info", "CustomInfo"] +CMOR_TABLES: dict[str, InfoBase] = {} +"""dict of str, obj: CMOR info objects. -CMOR_TABLES: dict[str, CMORTable] = {} -"""dict of str, obj: CMOR info objects.""" +.. deprecated:: 2.14.0 + The global ``CMOR_TABLES`` dictionary is deprecated and will be removed in + ESMValCore v2.16.0. Please use :func:`~esmvalcore.cmor.table.get_tables` + to access the CMOR tables instead. + +.. note:: + If this dictionary is empty, it can be populated by loading the global + configuration by importing the :mod:`esmvalcore.config` module. +""" _CMOR_KEYS = ( "standard_name", @@ -37,18 +52,39 @@ ) -def _update_cmor_facets(facets): +def _get_institutes(project: str, dataset: str) -> list[str]: + """Return the institutes from the controlled vocabulary given the dataset name.""" + try: + return CMOR_TABLES[project].institutes[dataset] # type: ignore[attr-defined] + except (KeyError, AttributeError): + return [] + + +def _get_activity( + project: str, + exp: str | list[str], +) -> str | list[str] | None: + """Return the activity from the controlled vocabulary given the experiment name.""" + try: + if isinstance(exp, list): + return [CMOR_TABLES[project].activities[value][0] for value in exp] # type: ignore[attr-defined] + return CMOR_TABLES[project].activities[exp][0] # type: ignore[attr-defined] + except (KeyError, AttributeError): + return None + + +def _update_cmor_facets(facets: Facets) -> None: """Update `facets` with information from CMOR table.""" - project = facets["project"] - mip = facets["mip"] - short_name = facets["short_name"] - derive = facets.get("derive", False) + project: str = facets["project"] # type: ignore[assignment] + mip: str = facets["mip"] # type: ignore[assignment] + short_name: str = facets["short_name"] # type: ignore[assignment] + derive: bool = facets.get("derive", False) # type: ignore[assignment] table = CMOR_TABLES.get(project) if table: table_entry = table.get_variable( mip, short_name, - branding_suffix=facets.get("branding_suffix"), + branding_suffix=facets.get("branding_suffix"), # type: ignore[arg-type] derived=derive, ) else: @@ -71,6 +107,14 @@ def _update_cmor_facets(facets): key, facets, ) + if "dataset" in facets and "institute" not in facets: + institute = _get_institutes(project, facets["dataset"]) # type: ignore[arg-type] + if institute: + facets["institute"] = institute + if "exp" in facets and "activity" not in facets: + activity = _get_activity(project, facets["exp"]) # type: ignore[arg-type] + if activity: + facets["activity"] = activity def _get_mips(project: str, short_name: str) -> list[str]: @@ -157,6 +201,12 @@ def get_var_info( def read_cmor_tables(cfg_developer: Path | None = None) -> None: """Read cmor tables required in the configuration. + .. deprecated:: 2.14.0 + + The config-developer.yml file based configuration is deprecated and + will no longer be supported in ESMValCore v2.16.0. Please use + :func:`~esmvalcore.cmor.table.load_cmor_tables` instead of this function. + Parameters ---------- cfg_developer: @@ -173,16 +223,15 @@ def read_cmor_tables(cfg_developer: Path | None = None) -> None: msg = "cfg_developer is not a Path-like object, got " raise TypeError(msg, cfg_developer) mtime = cfg_developer.stat().st_mtime - cmor_tables = _read_cmor_tables(cfg_developer, mtime) - CMOR_TABLES.clear() - CMOR_TABLES.update(cmor_tables) + global CMOR_TABLES # noqa: PLW0603 + CMOR_TABLES = _read_cmor_tables(cfg_developer, mtime) @lru_cache def _read_cmor_tables( cfg_file: Path, mtime: float, # noqa: ARG001 -) -> dict[str, CMORTable]: +) -> dict[str, InfoBase]: """Read cmor tables required in the configuration. Parameters @@ -201,7 +250,7 @@ def _read_cmor_tables( with open(var_alt_names_file, encoding="utf-8") as yfile: alt_names = yaml.safe_load(yfile) - cmor_tables: dict[str, CMORTable] = {} + cmor_tables: dict[str, InfoBase] = {} # Try to infer location for custom tables from config-developer.yml file, # if not possible, use default location @@ -264,33 +313,176 @@ def _read_table(cfg_developer, table, install_dir, custom, alt_names): raise ValueError(msg) -class InfoBase: - """Base class for all table info classes. +_TABLE_CACHE: dict[str, InfoBase] = {} +"""The CMOR tables are cached for faster access.""" + - This uses CMOR 3 json format +def clear_table_cache() -> None: + """Clear the CMOR table cache.""" + _TABLE_CACHE.clear() + + +def get_tables( + session: Session | Config, + project: str, +) -> InfoBase: + """Get the CMOR tables for a project. Parameters ---------- - default: object - Default table to look variables on if not found + session: + The configuration. + project: + The project to load a CMOR table for. + """ + if project not in session["projects"]: + msg = f"Unknown project '{project}', please configure it under 'projects'." + raise ValueError(msg) + + kwargs = ( + session["projects"][project] + .get( + "cmor_table", + { + "type": "esmvalcore.cmor.table.NoInfo", + }, + ) + .copy() + ) + if "type" not in kwargs: + msg = ( + f"Missing CMOR table 'type' in configuration of project {project}. " + f"Current configuration is:\n{yaml.safe_dump(kwargs)}" + ) + raise ValueError(msg) + cache_key = str(kwargs) + if cache_key not in _TABLE_CACHE: + import_error_messsage = ( + "Please check your configuration. " + f"Current configuration is:\n\n{ + yaml.safe_dump( + {'projects': {project: {'cmor_table': kwargs}}}, + ) + }\n" + "Typical good values for 'type' are " + "'esmvalcore.cmor.table.CMIP6Info' for CMIP6-style CMOR tables or " + "'esmvalcore.cmor.table.CMIP5Info' for CMIP5-style CMOR tables." + ) + module_name, cls_name = kwargs.pop("type").rsplit(".", 1) + try: + module = importlib.import_module(module_name) + except ModuleNotFoundError as exc: + msg = ( + f"Failed to import module '{module_name}' for CMOR table of " + f"project '{project}'. {import_error_messsage}" + ) + raise InvalidConfigParameter(msg) from exc + try: + cls = getattr(module, cls_name) + except AttributeError as exc: + msg = ( + f"Class '{cls_name}' for reading CMOR table of project " + f"'{project}' does not exist in module '{module_name}'. " + f"{import_error_messsage}" + ) + raise InvalidConfigParameter(msg) from exc + tables = cls(**kwargs) + if not isinstance(tables, InfoBase): + msg = ( + "`type` should be a subclass `esmvalcore.cmor.table.InfoBase`, " + f"but your configuration for project '{project}' contains " + f"'{tables}' of type: '{type(tables)}'. {import_error_messsage}" + ) + raise TypeError(msg) + _TABLE_CACHE[cache_key] = tables + + return _TABLE_CACHE[cache_key] - alt_names: list[list[str]] - List of known alternative names for variables - strict: bool - If False, will look for a variable in other tables if it can not be - found in the requested one +class InfoBase: + """Base class for all CMOR table info classes. + + Parameters + ---------- + default: + Default table to look variables on if not found. + + .. deprecated:: 2.14.0 + + The ``default`` parameter is deprecated and will be removed in + ESMValCore v2.16.0. Please use the ``paths`` parameter instead + to aggregate multiple tables. + + alt_names: + List of known alternative names for variables. If no value is provided, + the default values from the installed copy of + `variable_alt_names.yml `_ + will be used. + + strict: + If :obj:`False`, the function :meth:`~esmvalcore.cmor.table.InfoBase.get_variable` + will look for a variable in other tables if it can not be found in the + table specified by ``mip`` in the :ref:`recipe ` or :class:`~esmvalcore.dataset.Dataset`. + + paths: + A list of paths to CMOR tables. The path can be relative to the built-in + tables in the + `esmvalcore/cmor/tables `_ + directory, or any other path. The built-in tables will be used if the + path is relative and exists in the built-in tables directory. """ - def __init__(self, default, alt_names, strict): + def __init__( + self, + default: CustomInfo | None = None, + alt_names: list[list[str]] | None = None, + strict: bool = True, + paths: Iterable[Path] = (), + ) -> None: + # Configure the paths to the CMOR tables. + builtin_tables_path = Path(__file__).parent / "tables" + paths = tuple(Path(os.path.expandvars(p)).expanduser() for p in paths) + self.paths = tuple( + builtin_tables_path / p + if (builtin_tables_path / p).is_dir() + else p + for p in paths + ) + """A list of paths to CMOR tables.""" + for path in self.paths: + if not path.is_dir(): + raise NotADirectoryError(path) + + # Configure the alternative names. if alt_names is None: - alt_names = "" - self.default = default + alt_names_path = Path(__file__).parent / "variable_alt_names.yml" + alt_names = yaml.safe_load( + alt_names_path.read_text(encoding="utf-8"), + ) self.alt_names = alt_names + """List of known alternative names for variables.""" + self.coords: dict[str, CoordinateInfo] = {} + """The coordinates defined in these tables.""" + self.default = default + """ + Default table to look variables on if not found. + + .. deprecated:: 2.14.0 + + The ``default`` attribute is deprecated and will be removed in + ESMValCore v2.16.0. + """ self.strict = strict - self.tables = {} + """If False, will look for a variable in other tables if it can not be + found in the requested one. + """ + self.tables: dict[str, TableInfo] = {} + """A mapping from table names to :class:`TableInfo` objects.""" + + def __repr__(self) -> str: + return f"{self.__class__.__name__}(paths={list(self.paths)}, strict={self.strict}, alt_names={self.alt_names})" - def get_table(self, table): + def get_table(self, table: str) -> TableInfo | None: """Search and return the table info. Parameters @@ -319,12 +511,18 @@ def get_variable( Parameters ---------- table_name: - Table name, i.e., the variable's MIP. + Table name, i.e., the ``mip`` in the :ref:`recipe ` or + :class:`~esmvalcore.dataset.Dataset`, e.g. ``"Omon"`` for CMIP6 or + ``"ocean"`` for CMIP7. short_name: - Variable's short name. + Variable's short name, e.g. ``"tos"`` for sea surface temperature. branding_suffix: A suffix that will be appended to ``short_name`` when looking up the - variable in the CMOR table. + variable in the CMOR table, e.g. a + `CMIP7 branding suffix `__, + could be ``"tavg-u-hxy-sea"``, which defines the temporal average + at an undefined vertical level on a horizontal grid where non-sea + points are masked. derived: Variable is derived. Information retrieval for derived variables always looks in the default tables (usually, the custom tables) if @@ -367,7 +565,7 @@ def get_variable( # If that didn't work either, look in default table if # cmor_strict=False or derived=True - if not var_info: + if not var_info and self.default is not None: var_info = self._look_in_default( derived, alt_names_list, @@ -385,6 +583,7 @@ def get_variable( def _look_in_default(self, derived, alt_names_list, table_name): """Look for variable in default table.""" + # TODO: remove in v2.16.0 var_info = None if not self.strict or derived: for alt_names in alt_names_list: @@ -433,59 +632,118 @@ def _look_all_tables(self, alt_names): class CMIP6Info(InfoBase): - """Class to read CMIP6-like data request. + """Class to read CMIP6-like CMOR tables. - This uses CMOR 3 json format + This class reads CMOR 3 json format tables. Parameters ---------- - cmor_tables_path: str - Path to the folder containing the Tables folder with the json files + cmor_tables_path: + The path to a directory with subdirectory "Tables" where the CMOR tables + are located. + + .. deprecated:: 2.14.0 + + The ``cmor_tables_path`` parameter is deprecated and will be removed in + ESMValCore v2.16.0. Please use the ``paths`` parameter instead. + + default: + Default table to look variables on if not found. + + .. deprecated:: 2.14.0 + + The ``default`` parameter is deprecated and will be removed in + ESMValCore v2.16.0. Please use the ``paths`` parameter instead + to aggregate multiple tables. - default: object - Default table to look variables on if not found + alt_names: + List of known alternative names for variables. If no value is provided, + the default values from the installed copy of + `variable_alt_names.yml`_ will be used. + + strict: + If :obj:`False`, the function :meth:`~esmvalcore.cmor.table.InfoBase.get_variable` + will look for a variable in other tables if it can not be found in the + table specified by ``mip`` in the :ref:`recipe ` or + :class:`~esmvalcore.dataset.Dataset`. + + default_table_prefix: + If the table_id contains a prefix, it can be specified here. + + .. deprecated:: 2.14.0 + + The ``default_table_prefix`` parameter is deprecated and will be removed in + ESMValCore v2.16.0. + + paths: + A list of paths to CMOR tables. The path can be relative to the built-in + tables in the + `esmvalcore/cmor/tables `_ + directory, or any other path. The built-in tables will be used if the + path is relative and exists in the built-in tables directory. Only files + with the extension ``.json`` in the specified paths will be read as a + CMOR tables, any other files will be ignored. - strict: bool - If False, will look for a variable in other tables if it can not be - found in the requested one """ def __init__( self, - cmor_tables_path, - default=None, - alt_names=None, - strict=True, - default_table_prefix="", - ): - super().__init__(default, alt_names, strict) - cmor_tables_path = self._get_cmor_path(cmor_tables_path) - - self._cmor_folder = os.path.join(cmor_tables_path, "Tables") - if glob.glob(os.path.join(self._cmor_folder, "*_CV.json")): - self._load_controlled_vocabulary() + cmor_tables_path: str | None = None, + default: CustomInfo | None = None, + alt_names: list[list[str]] | None = None, + strict: bool = True, + default_table_prefix: str = "", + paths: Iterable[Path] = (), + ) -> None: + if cmor_tables_path is not None: + # Support cmor_tables_path for backward compatibility. + # TODO: remove in v2.16.0 + tables_path = Path(self._get_cmor_path(cmor_tables_path)) + if (tables_path / "tables").exists(): + # Support CMIP7 which uses a lowercase "tables" subdirectory. + cmor_folder = tables_path / "tables" + else: + cmor_folder = tables_path / "Tables" + paths = (*tuple(paths), cmor_folder) + super().__init__(default, alt_names, strict, paths=paths) self.default_table_prefix = default_table_prefix + """ + If the table_id contains a prefix, it can be specified here. - self.var_to_freq = {} + .. deprecated:: 2.14.0 - self._load_coordinates() - for json_file in glob.glob(os.path.join(self._cmor_folder, "*.json")): - if "CV_test" in json_file or "grids" in json_file: - continue - try: - self._load_table(json_file) - except Exception: - msg = f"Exception raised when loading {json_file}" - # Logger may not be ready at this stage - if logger.handlers: - logger.error(msg) - else: - print(msg) # noqa: T201 - raise + The ``default_table_prefix`` attribute is deprecated and will be + removed in ESMValCore v2.16.0. + """ + self.var_to_freq: dict[str, dict[str, str]] = {} + self.activities: dict[str, list[str]] = {} + """A mapping from ``exp`` to ``activity`` from the controlled vocabulary.""" + self.institutes: dict[str, list[str]] = {} + """A mapping from ``dataset`` to ``institute`` from the controlled vocabulary.""" + + for path in self.paths: + if not any(path.glob("*.json")): + msg = f"No CMOR tables found in {path}" + raise ValueError(msg) + self._load_controlled_vocabulary(path) + self._load_coordinates(path) + for json_file in glob.glob(os.path.join(path, "*.json")): + if "CV_test" in json_file or "grids" in json_file: + continue + try: + self._load_table(json_file) + except Exception: + msg = f"Exception raised when loading {json_file}" + # Logger may not be ready at this stage + if logger.handlers: + logger.error(msg) + else: + print(msg) # noqa: T201 + raise @staticmethod - def _get_cmor_path(cmor_tables_path): + def _get_cmor_path(cmor_tables_path: str) -> str: if os.path.isdir(cmor_tables_path): return cmor_tables_path cwd = os.path.dirname(os.path.realpath(__file__)) @@ -500,17 +758,19 @@ def _load_table(self, json_file): raw_data = json.loads(inf.read()) if not self._is_table(raw_data): return - table = TableInfo() header = raw_data["Header"] - table.name = header["table_id"].split(" ")[-1] - self.tables[table.name] = table + table_name = header["table_id"].split(" ")[-1] + if table_name not in self.tables: + table = TableInfo() + table.name = table_name + self.tables[table_name] = table + table = self.tables[table_name] generic_levels = header["generic_levels"].split() - table.frequency = header.get("frequency", "") self.var_to_freq[table.name] = {} for var_name, var_data in raw_data["variable_entry"].items(): - var = VariableInfo("CMIP6", var_name) + var = VariableInfo("CMIP6") var.read_json(var_data, table.frequency) self._assign_dimensions(var, generic_levels) table[var_name] = var @@ -520,7 +780,6 @@ def _load_table(self, json_file): var_freqs = (var.frequency for var in table.values()) table_freq, _ = Counter(var_freqs).most_common(1)[0] table.frequency = table_freq - self.tables[table.name] = table def _assign_dimensions(self, var, generic_levels): for dimension in var.dimensions: @@ -544,10 +803,9 @@ def _assign_dimensions(self, var, generic_levels): var.coordinates[dimension] = coord - def _load_coordinates(self): - self.coords = {} + def _load_coordinates(self, path: Path) -> None: for json_file in glob.glob( - os.path.join(self._cmor_folder, "*coordinate*.json"), + os.path.join(path, "*coordinate*.json"), ): with open(json_file, encoding="utf-8") as inf: table_data = json.loads(inf.read()) @@ -556,11 +814,9 @@ def _load_coordinates(self): coord.read_json(table_data["axis_entry"][coord_name]) self.coords[coord_name] = coord - def _load_controlled_vocabulary(self): - self.activities = {} - self.institutes = {} + def _load_controlled_vocabulary(self, path: Path) -> None: for json_file in glob.glob( - os.path.join(self._cmor_folder, "*_CV.json"), + os.path.join(path, "*_CV.json"), ): with open(json_file, encoding="utf-8") as inf: table_data = json.loads(inf.read()) @@ -580,17 +836,17 @@ def _load_controlled_vocabulary(self): except (KeyError, AttributeError): pass - def get_table(self, table): + def get_table(self, table: str) -> TableInfo | None: """Search and return the table info. Parameters ---------- - table: str + table: Table name Returns ------- - TableInfo + : Return the TableInfo object for the requested table if found, returns None if not """ @@ -606,6 +862,49 @@ def _is_table(table_data): return "Header" in table_data +class Obs4MIPsInfo(CMIP6Info): + """Class to read obs4MIPs-like CMOR tables. + + Parameters + ---------- + alt_names: + List of known alternative names for variables. If no value is provided, + the default values from the installed copy of + `variable_alt_names.yml`_ will be used. + + strict: + If :obj:`False`, the function :meth:`~esmvalcore.cmor.table.InfoBase.get_variable` + will look for a variable in other tables if it can not be found in the + table specified by ``mip`` in the :ref:`recipe ` or + :class:`~esmvalcore.dataset.Dataset`. + + paths: + A list of paths to CMOR tables. The path can be relative to the built-in + tables in the + `esmvalcore/cmor/tables `_ + directory, or any other path. The built-in tables will be used if the + path is relative and exists in the built-in tables directory. + """ + + def __init__( + self, + alt_names: list[list[str]] | None = None, + strict: bool = True, + paths: Iterable[Path] = (), + ) -> None: + super().__init__( + alt_names=alt_names, + strict=strict, + paths=paths, + ) + # Remove the prefix from the table_id. + table_id_prefix = "obs4MIPs_" + for name in list(self.tables): + if name.startswith(table_id_prefix): + table = self.tables.pop(name) + self.tables[name[len(table_id_prefix) :]] = table + + @total_ordering class TableInfo(dict): """Container class for storing a CMOR table.""" @@ -614,8 +913,11 @@ def __init__(self, *args, **kwargs): """Create a new TableInfo object for storing VariableInfo objects.""" super().__init__(*args, **kwargs) self.name = "" + """Table name.""" self.frequency = "" + """Table frequency (if defined).""" self.realm = "" + """Table realm (if defined).""" def __eq__(self, other): return (self.name, self.frequency, self.realm) == ( @@ -689,17 +991,33 @@ def _read_json_list_variable(self, parameter): class VariableInfo(JsonInfo): """Class to read and store variable information.""" - def __init__(self, table_type, short_name): + def __init__( + self, + table_type: str = "", + short_name: str = "", + ) -> None: """Class to read and store variable information. Parameters ---------- - short_name: str + table_type: + Type of table (e.g., CMIP5, CMIP6). + + .. deprecated:: 2.14.0 + + The ``table_type`` parameter is deprecated and will be removed + in ESMValCore v2.16.0. + short_name: Variable's short name. + + .. deprecated:: 2.14.0 + + The ``short_name`` parameter is deprecated and will be removed + in ESMValCore v2.16.0. """ super().__init__() self.table_type = table_type - self.modeling_realm = [] + self.modeling_realm: list[str] = [] """Modeling realm""" self.short_name = short_name """Short name""" @@ -718,9 +1036,9 @@ def __init__(self, table_type, short_name): self.positive = "" """Increasing direction""" - self.dimensions = [] + self.dimensions: list[str] = [] """List of dimensions""" - self.coordinates = {} + self.coordinates: dict[str, CoordinateInfo] = {} """Coordinates This is a dict with the names of the dimensions as keys and @@ -729,7 +1047,10 @@ def __init__(self, table_type, short_name): self._json_data = None - def copy(self): + def __repr__(self) -> str: + return f"<{self.__class__.__name__} defining variable '{self.short_name}'>" + + def copy(self) -> Self: """Return a shallow copy of VariableInfo. Returns @@ -739,17 +1060,17 @@ def copy(self): """ return copy.copy(self) - def read_json(self, json_data, default_freq): + def read_json(self, json_data: dict, default_freq: str) -> None: """Read variable information from json. Non-present options will be set to empty Parameters ---------- - json_data: dict + json_data: Dictionary created by the json reader containing variable information. - default_freq: str + default_freq: Default frequency to use if it is not defined at variable level. """ self._json_data = json_data @@ -785,12 +1106,12 @@ def has_coord_with_standard_name(self, standard_name: str) -> bool: Parameters ---------- - standard_name: str + standard_name: Standard name to be checked. Returns ------- - bool + : `True` if there is at least one coordinate with the given `standard_name`, `False` if not. @@ -804,18 +1125,19 @@ def has_coord_with_standard_name(self, standard_name: str) -> bool: class CoordinateInfo(JsonInfo): """Class to read and store coordinate information.""" - def __init__(self, name): + def __init__(self, name: str) -> None: """Class to read and store coordinate information. Parameters ---------- - name: str + name: coordinate's name """ super().__init__() self.name = name + """Name of the coordinate entry in the CMOR table.""" self.generic_level = False - self.generic_lev_coords = {} + self.generic_lev_coords: dict[str, CoordinateInfo] = {} self.axis = "" """Axis""" @@ -837,7 +1159,7 @@ def __init__(self, name): """Units""" self.stored_direction = "" """Direction in which the coordinate increases""" - self.requested = [] + self.requested: list[str] = [] """Values requested""" self.valid_min = "" """Minimum allowed value""" @@ -864,7 +1186,7 @@ def read_json(self, json_data): self.axis = self._read_json_variable("axis") self.value = self._read_json_variable("value") self.out_name = self._read_json_variable("out_name") - self.var_name = self._read_json_variable("var_name") + self.var_name = self._read_json_variable("out_name") self.standard_name = self._read_json_variable("standard_name") self.long_name = self._read_json_variable("long_name") self.units = self._read_json_variable("units") @@ -877,58 +1199,97 @@ def read_json(self, json_data): class CMIP5Info(InfoBase): - """Class to read CMIP5-like data request. + """Class to read CMIP5-like CMOR tables. + + This class reads CMOR 2 format tables. Parameters ---------- - cmor_tables_path: str - Path to the folder containing the Tables folder with the json files + cmor_tables_path: + The path to a directory with subdirectory "Tables" where the CMOR tables + are located. + + .. deprecated:: 2.14.0 + + The ``cmor_tables_path`` parameter is deprecated and will be removed in + ESMValCore v2.16.0. Please use the ``paths`` parameter instead. + + default: + Default table to look variables on if not found. + + .. deprecated:: 2.14.0 - default: object - Default table to look variables on if not found + The ``default`` parameter is deprecated and will be removed in + ESMValCore v2.16.0. Please use the ``paths`` parameter instead + to aggregate multiple tables. + + alt_names: + List of known alternative names for variables. If no value is provided, + the default values from the installed copy of + `variable_alt_names.yml`_ will be used. + + strict: + If :obj:`False`, the function :meth:`~esmvalcore.cmor.table.InfoBase.get_variable` + will look for a variable in other tables if it can not be found in the + table specified by ``mip`` in the :ref:`recipe ` or + :class:`~esmvalcore.dataset.Dataset`. + + default_table_prefix: + If the table_id contains a prefix, it can be specified here. + + .. deprecated:: 2.14.0 + + The ``default_table_prefix`` parameter is deprecated and will be removed in + ESMValCore v2.16.0. + + paths: + A list of paths to CMOR tables. The path can be relative to the built-in + tables in the + `esmvalcore/cmor/tables `_ + directory, or any other path. The built-in tables will be used if the + path is relative and exists in the built-in tables directory. Any file + in the specified paths will be read as a CMOR table. - strict: bool - If False, will look for a variable in other tables if it can not be - found in the requested one """ def __init__( self, - cmor_tables_path, - default=None, - alt_names=None, - strict=True, - ): - super().__init__(default, alt_names, strict) - cmor_tables_path = self._get_cmor_path(cmor_tables_path) - - self._cmor_folder = os.path.join(cmor_tables_path, "Tables") - if not os.path.isdir(self._cmor_folder): - raise OSError( - errno.ENOTDIR, - "CMOR tables path is not a directory", - self._cmor_folder, - ) - - self.strict = strict - self.tables = {} - self.coords = {} - self._current_table = None - self._last_line_read = None - - for table_file in glob.glob(os.path.join(self._cmor_folder, "*")): - if "_grids" in table_file: - continue - try: - self._load_table(table_file) - except Exception: - msg = f"Exception raised when loading {table_file}" - # Logger may not be ready at this stage - if logger.handlers: - logger.error(msg) - else: - print(msg) # noqa: T201 - raise + cmor_tables_path: str | None = None, + default: CustomInfo | None = None, + alt_names: list[list[str]] | None = None, + strict: bool = True, + paths: Iterable[Path] = (), + ) -> None: + if cmor_tables_path is not None: + # Support cmor_tables_path for backward compatibility. + # TODO: remove in v2.16.0 + cmor_tables_path = self._get_cmor_path(cmor_tables_path) + cmor_folder = Path(cmor_tables_path) / "Tables" + paths = (*tuple(paths), cmor_folder) + super().__init__(default, alt_names, strict, paths=paths) + + self._current_table: TextIOWrapper | None = None + self._last_line_read = ("", "") + + for path in self.paths: + for table_file in sorted( + glob.glob(os.path.join(path, "*")), + # Read coordinate files before variable files so we can link the + # variables with the coordinates. + key=lambda filename: "coordinate" not in filename, + ): + if "_grids" in table_file: + continue + try: + self._load_table(table_file) + except Exception: + msg = f"Exception raised when loading {table_file}" + # Logger may not be ready at this stage + if logger.handlers: + logger.error(msg) + else: + print(msg) # noqa: T201 + raise @staticmethod def _get_cmor_path(cmor_tables_path): @@ -937,25 +1298,21 @@ def _get_cmor_path(cmor_tables_path): cwd = os.path.dirname(os.path.realpath(__file__)) return os.path.join(cwd, "tables", cmor_tables_path) - def _load_table(self, table_file, table_name=""): - if table_name and table_name in self.tables: - # special case used for updating a table with custom variable file - table = self.tables[table_name] + def _load_table(self, table_file: str) -> None: + table = self._read_table_file(table_file) + if table.name in self.tables: + self.tables[table.name].update(table) else: - # default case: table name is first line of table file - table = None - - self._read_table_file(table_file, table) + self.tables[table.name] = table - def _read_table_file(self, table_file, table=None): + def _read_table_file(self, table_file: str) -> TableInfo: + table = TableInfo() with open(table_file, encoding="utf-8") as self._current_table: self._read_line() while True: key, value = self._last_line_read if key == "table_id": - table = TableInfo() table.name = value[len("Table ") :] - self.tables[table.name] = table elif key == "frequency": table.frequency = value elif key == "modeling_realm": @@ -973,7 +1330,8 @@ def _read_table_file(self, table_file, table=None): table[value] = self._read_variable(value, table.frequency) continue if not self._read_line(): - return + break + return table def _read_line(self): line = self._current_table.readline() @@ -1008,8 +1366,8 @@ def _read_coordinate(self, value): setattr(coord, key, value) return coord - def _read_variable(self, short_name, frequency): - var = VariableInfo("CMIP5", short_name) + def _read_variable(self, entry_name, frequency): + var = VariableInfo(table_type="CMIP5") var.frequency = frequency while self._read_line(): key, value = self._last_line_read @@ -1021,21 +1379,25 @@ def _read_variable(self, short_name, frequency): setattr(var, key, value) elif key == "out_name": var.short_name = value + if not var.short_name: + # Some of our custom CMIP5 table entries are missing the `out_name` field. + # In that case, we assume the entry name is the same as short_name. + var.short_name = entry_name for dim in var.dimensions: var.coordinates[dim] = self.coords[dim] return var - def get_table(self, table): + def get_table(self, table: str) -> TableInfo | None: """Search and return the table info. Parameters ---------- - table: str + table: Table name Returns ------- - TableInfo + : Return the TableInfo object for the requested table if found, returns None if not """ @@ -1043,28 +1405,64 @@ def get_table(self, table): class CMIP3Info(CMIP5Info): - """Class to read CMIP3-like data request. + """Class to read CMIP3-like CMOR tables. Parameters ---------- - cmor_tables_path: str - Path to the folder containing the Tables folder with the json files + cmor_tables_path: + The path to a directory with subdirectory "Tables" where the CMOR tables + are located. + + .. deprecated:: 2.14.0 + + The ``cmor_tables_path`` parameter is deprecated and will be removed in + ESMValCore v2.16.0. Please use the ``paths`` parameter instead. + + default: + Default table to look variables on if not found. + + .. deprecated:: 2.14.0 - default: object - Default table to look variables on if not found + The ``default`` parameter is deprecated and will be removed in + ESMValCore v2.16.0. Please use the ``paths`` parameter instead + to aggregate multiple tables. + + alt_names: + List of known alternative names for variables. If no value is provided, + the default values from the installed copy of + `variable_alt_names.yml`_ will be used. + + strict: + If :obj:`False`, the function :meth:`~esmvalcore.cmor.table.InfoBase.get_variable` + will look for a variable in other tables if it can not be found in the + table specified by ``mip`` in the :ref:`recipe ` or + :class:`~esmvalcore.dataset.Dataset`. + + default_table_prefix: + If the table_id contains a prefix, it can be specified here. + + .. deprecated:: 2.14.0 + + The ``default_table_prefix`` parameter is deprecated and will be removed in + ESMValCore v2.16.0. + + paths: + A list of paths to CMOR tables. The path can be relative to the built-in + tables in the + `esmvalcore/cmor/tables `_ + directory, or any other path. The built-in tables will be used if the + path is relative and exists in the built-in tables directory. Any file + in the specified paths will be read as a CMOR table. - strict: bool - If False, will look for a variable in other tables if it can not be - found in the requested one """ - def _read_table_file(self, table_file, table=None): + def _read_table_file(self, table_file: str) -> TableInfo: for dim in ("zlevel",): coord = CoordinateInfo(dim) coord.generic_level = True coord.axis = "Z" self.coords[dim] = coord - super()._read_table_file(table_file, table) + return super()._read_table_file(table_file) def _read_coordinate(self, value): coord = super()._read_coordinate(value) @@ -1073,16 +1471,21 @@ def _read_coordinate(self, value): coord.var_name = coord.name return coord - def _read_variable(self, short_name, frequency): - var = super()._read_variable(short_name, frequency) - var.frequency = None - var.modeling_realm = None + def _read_variable(self, entry_name, frequency): + var = super()._read_variable(entry_name, frequency) + var.frequency = "" + var.modeling_realm = [] return var class CustomInfo(CMIP5Info): """Class to read custom var info for ESMVal. + .. deprecated:: 2.14.0 + + This class is deprecated and will be removed in ESMValCore v2.16.0. + Please use :class:`~esmvalcore.cmor.tables.table.CMIP5Info` instead. + Parameters ---------- cmor_tables_path: @@ -1096,28 +1499,34 @@ def __init__(self, cmor_tables_path: str | Path | None = None) -> None: """Initialize class member.""" self.coords = {} self.tables = {} - self.var_to_freq: dict[str, dict] = {} + self.var_to_freq: dict[str, dict[str, str]] = {} table = TableInfo() table.name = "custom" self.tables[table.name] = table # First, read default custom tables from repository - self._cmor_folder = self._get_cmor_path("custom") - self._read_table_dir(self._cmor_folder) + self.paths = ( + Path(self._get_cmor_path("old-custom-coordinates")), + Path(self._get_cmor_path("cmip5-custom")), + ) # Second, if given, update default tables with user-defined custom # tables if cmor_tables_path is not None: - self._user_table_folder = self._get_cmor_path(cmor_tables_path) - if not os.path.isdir(self._user_table_folder): + user_table_folder = Path(self._get_cmor_path(cmor_tables_path)) + if not user_table_folder.is_dir(): msg = ( - f"Custom CMOR tables path {self._user_table_folder} is " + f"Custom CMOR tables path {user_table_folder} is " f"not a directory" ) raise ValueError(msg) - self._read_table_dir(self._user_table_folder) - else: - self._user_table_folder = None + self.paths += (user_table_folder,) + + for path in self.paths: + self._read_table_dir(str(path)) + + def __repr__(self) -> str: + return f"{self.__class__.__name__}(paths={list(self.paths)})" def _read_table_dir(self, table_dir: str) -> None: """Read CMOR tables from directory.""" @@ -1131,7 +1540,7 @@ def _read_table_dir(self, table_dir: str) -> None: if dat_file == coordinates_file: continue try: - self._read_table_file(dat_file) + self._load_table(dat_file) except Exception: msg = f"Exception raised when loading {dat_file}" # Logger may not be ready at this stage @@ -1153,13 +1562,19 @@ def get_variable( Parameters ---------- - table: - Table name. Ignored for custom tables. + table_name: + Table name, i.e., the ``mip`` in the :ref:`recipe ` or + :class:`~esmvalcore.dataset.Dataset`, e.g. ``"Omon"`` for CMIP6 or + ``"ocean"`` for CMIP7. short_name: - Variable's short name. + Variable's short name, e.g. ``"tos"`` for sea surface temperature. branding_suffix: A suffix that will be appended to ``short_name`` when looking up the - variable in the CMOR table. Ignored for custom tables. + variable in the CMOR table, e.g. a + `CMIP7 branding suffix `__, + could be ``"tavg-u-hxy-sea"``, which defines the temporal average + at an undefined vertical level on a horizontal grid where non-sea + points are masked. derived: Variable is derived. Info retrieval for derived variables always looks on the default tables if variable is not found in the @@ -1174,12 +1589,10 @@ def get_variable( """ return self.tables["custom"].get(short_name, None) - def _read_table_file( - self, - table_file: str, - table: TableInfo | None = None, # noqa: ARG002 - ) -> None: + def _read_table_file(self, table_file: str) -> TableInfo: """Read a single table file.""" + table = TableInfo() + table.name = "custom" with open(table_file, encoding="utf-8") as self._current_table: self._read_line() while True: @@ -1194,14 +1607,58 @@ def _read_table_file( self.coords[value] = self._read_coordinate(value) continue elif key == "variable_entry": - self.tables["custom"][value] = self._read_variable( - value, - "", - ) + table[value] = self._read_variable(value, "") continue if not self._read_line(): - return + return table + + +class NoInfo(InfoBase): + """Table that can be used for projects that do not provide a CMOR table.""" + + def __init__(self) -> None: + super().__init__() + + def __repr__(self) -> str: + return f"{self.__class__.__name__}()" + + def get_variable( + self, + table_name: str, # noqa: ARG002 + short_name: str, + *, + branding_suffix: str | None = None, # noqa: ARG002 + derived: bool = False, # noqa: ARG002 + ) -> VariableInfo | None: + """Search and return the variable information. + + Parameters + ---------- + table_name: + Table name, i.e., the ``mip`` in the :ref:`recipe ` or + :class:`~esmvalcore.dataset.Dataset`, e.g. ``"Omon"`` for CMIP6 or + ``"ocean"`` for CMIP7. + short_name: + Variable's short name, e.g. ``"tos"`` for sea surface temperature. + branding_suffix: + A suffix that will be appended to ``short_name`` when looking up the + variable in the CMOR table, e.g. a + `CMIP7 branding suffix `__, + could be ``"tavg-u-hxy-sea"``, which defines the temporal average + at an undefined vertical level on a horizontal grid where non-sea + points are masked. + derived: + Variable is derived. Information retrieval for derived variables + always looks in the default tables (usually, the custom tables) if + variable is not found in the requested table. + Returns + ------- + VariableInfo | None + `VariableInfo` object for the requested variable if found, ``None`` + otherwise. -# Load the default tables on initializing the module. -read_cmor_tables() + """ + vardef = VariableInfo() + vardef.short_name = short_name + return vardef diff --git a/esmvalcore/cmor/tables/cmip3-custom/custom b/esmvalcore/cmor/tables/cmip3-custom/custom new file mode 100644 index 0000000000..d0b6cf54bc --- /dev/null +++ b/esmvalcore/cmor/tables/cmip3-custom/custom @@ -0,0 +1,45 @@ +table_id: Table custom + +!============ +variable_entry: lwcre +!============ +modeling_realm: atmos +!---------------------------------- +! Variable attributes: +!---------------------------------- +standard_name: +units: W m-2 +cell_methods: time: mean +cell_measures: area: areacella +long_name: TOA Longwave Cloud Radiative Effect +comment: at the top of the atmosphere (to be compared with satellite measurements) +!---------------------------------- +! Additional variable information: +!---------------------------------- +dimensions: longitude latitude time +type: real +positive: down +!---------------------------------- +! + +!============ +variable_entry: swcre +!============ +modeling_realm: atmos +!---------------------------------- +! Variable attributes: +!---------------------------------- +standard_name: +units: W m-2 +cell_methods: time: mean +cell_measures: area: areacella +long_name: TOA Shortwave Cloud Radiative Effect +comment: at the top of the atmosphere (to be compared with satellite measurements) +!---------------------------------- +! Additional variable information: +!---------------------------------- +dimensions: longitude latitude time +type: real +positive: down +!---------------------------------- +! diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_BC_tot.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_BC_tot.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_BC_tot.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_BC_tot.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_CFCl3.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_CFCl3.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_CFCl3.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_CFCl3.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_CH4.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_CH4.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_CH4.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_CH4.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_CO.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_CO.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_CO.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_CO.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_CO2.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_CO2.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_CO2.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_CO2.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_ClOX.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_ClOX.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_ClOX.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_ClOX.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_DU_tot.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_DU_tot.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_DU_tot.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_DU_tot.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_N2O.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_N2O.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_N2O.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_N2O.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_NH3.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_NH3.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_NH3.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_NH3.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_NO.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_NO.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_NO.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_NO.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_NO2.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_NO2.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_NO2.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_NO2.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_NOX.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_NOX.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_NOX.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_NOX.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_O3.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_O3.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_O3.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_O3.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_OH.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_OH.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_OH.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_OH.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_S.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_S.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_S.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_S.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_SO2.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_SO2.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_SO2.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_SO2.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_SO4mm_tot.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_SO4mm_tot.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_SO4mm_tot.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_SO4mm_tot.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_MP_SS_tot.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_SS_tot.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_MP_SS_tot.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_MP_SS_tot.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_agb.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_agb.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_agb.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_agb.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_alb.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_alb.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_alb.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_alb.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_albDiff.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_albDiff.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_albDiff.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_albDiff.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_albDiffiTr13.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_albDiffiTr13.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_albDiffiTr13.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_albDiffiTr13.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_amoc.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_amoc.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_amoc.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_amoc.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_asr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_asr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_asr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_asr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_awhea.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_awhea.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_awhea.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_awhea.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_bdalb.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_bdalb.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_bdalb.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_bdalb.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_bhalb.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_bhalb.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_bhalb.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_bhalb.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_ch4s.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_ch4s.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_ch4s.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_ch4s.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_chlora.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_chlora.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_chlora.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_chlora.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_clhmtisccp.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_clhmtisccp.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_clhmtisccp.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_clhmtisccp.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_clhtkisccp.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_clhtkisccp.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_clhtkisccp.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_clhtkisccp.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_clisccp.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_clisccp.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_clisccp.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_clisccp.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_cllmtisccp.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_cllmtisccp.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_cllmtisccp.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_cllmtisccp.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_clltkisccp.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_clltkisccp.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_clltkisccp.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_clltkisccp.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_clmmtisccp.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_clmmtisccp.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_clmmtisccp.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_clmmtisccp.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_clmtkisccp.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_clmtkisccp.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_clmtkisccp.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_clmtkisccp.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_cltStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_cltStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_cltStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_cltStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_co2s.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_co2s.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_co2s.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_co2s.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_ctotal.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_ctotal.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_ctotal.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_ctotal.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_dos.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_dos.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_dos.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_dos.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_dosStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_dosStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_dosStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_dosStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_dpn2o.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_dpn2o.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_dpn2o.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_dpn2o.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_et.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_et.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_et.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_et.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_etStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_etStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_etStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_etStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_fapar.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_fapar.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_fapar.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_fapar.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_gppStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_gppStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_gppStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_gppStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_hfns.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_hfns.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_hfns.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_hfns.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_hurStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_hurStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_hurStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_hurStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_husStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_husStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_husStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_husStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_iwpStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_iwpStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_iwpStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_iwpStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_lapserate.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_lapserate.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_lapserate.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_lapserate.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_lvp.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_lvp.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_lvp.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_lvp.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_lwcre.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_lwcre.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_lwcre.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_lwcre.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_lweGrace.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_lweGrace.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_lweGrace.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_lweGrace.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_lwp.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_lwp.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_lwp.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_lwp.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_lwpStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_lwpStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_lwpStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_lwpStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_n2oflux.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_n2oflux.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_n2oflux.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_n2oflux.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_n2os.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_n2os.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_n2os.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_n2os.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_netcre.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_netcre.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_netcre.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_netcre.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_od550aerStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_od550aerStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_od550aerStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_od550aerStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_od870aerStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_od870aerStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_od870aerStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_od870aerStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_ohc.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_ohc.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_ohc.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_ohc.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_prStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_prStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_prStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_prStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_prl.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_prl.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_prl.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_prl.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_prodlnox.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_prodlnox.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_prodlnox.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_prodlnox.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_ptype.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_ptype.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_ptype.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_ptype.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_qep.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_qep.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_qep.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_qep.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rlns.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rlns.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rlns.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rlns.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rlnst.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rlnst.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rlnst.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rlnst.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rlnstcs.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rlnstcs.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rlnstcs.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rlnstcs.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rlntcs.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rlntcs.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rlntcs.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rlntcs.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rluscs.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rluscs.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rluscs.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rluscs.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rlut.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rlut.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rlut.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rlut.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rlutcs.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rlutcs.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rlutcs.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rlutcs.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rsns.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rsns.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rsns.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rsns.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rsnst.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rsnst.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rsnst.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rsnst.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rsnstcs.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rsnstcs.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rsnstcs.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rsnstcs.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rsnstcsnorm.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rsnstcsnorm.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rsnstcsnorm.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rsnstcsnorm.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rsnt.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rsnt.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rsnt.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rsnt.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rsntcs.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rsntcs.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rsntcs.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rsntcs.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rsut.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rsut.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rsut.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rsut.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rsutcs.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rsutcs.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rsutcs.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rsutcs.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rtnt.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rtnt.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rtnt.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rtnt.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rx1day.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rx1day.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rx1day.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rx1day.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_rx5day.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_rx5day.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_rx5day.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_rx5day.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_siextent.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_siextent.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_siextent.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_siextent.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_sispeed.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_sispeed.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_sispeed.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_sispeed.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_sithick.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_sithick.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_sithick.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_sithick.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_sm.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_sm.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_sm.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_sm.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_sm1m.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_sm1m.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_sm1m.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_sm1m.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_smStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_smStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_smStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_smStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_soz.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_soz.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_soz.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_soz.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_swcre.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_swcre.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_swcre.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_swcre.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tasConf5.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tasConf5.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tasConf5.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tasConf5.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tasConf95.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tasConf95.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tasConf95.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tasConf95.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tasa.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tasa.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tasa.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tasa.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tasaga.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tasaga.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tasaga.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tasaga.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tcw.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tcw.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tcw.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tcw.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tnn.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tnn.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tnn.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tnn.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tosStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tosStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tosStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tosStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_toz.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_toz.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_toz.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_toz.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tozStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tozStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tozStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tozStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tro3prof.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tro3prof.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tro3prof.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tro3prof.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tro3profStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tro3profStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tro3profStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tro3profStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_troz.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_troz.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_troz.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_troz.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsDay.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsDay.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsDay.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsDay.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsLCDay.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLCDay.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsLCDay.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLCDay.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsLCNight.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLCNight.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsLCNight.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLCNight.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsLSSysErrDay.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLSSysErrDay.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsLSSysErrDay.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLSSysErrDay.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsLSSysErrNight.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLSSysErrNight.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsLSSysErrNight.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLSSysErrNight.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsLocalAtmErrDay.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLocalAtmErrDay.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsLocalAtmErrDay.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLocalAtmErrDay.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsLocalAtmErrNight.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLocalAtmErrNight.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsLocalAtmErrNight.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLocalAtmErrNight.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsLocalSfcErrDay.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLocalSfcErrDay.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsLocalSfcErrDay.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLocalSfcErrDay.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsLocalSfcErrNight.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLocalSfcErrNight.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsLocalSfcErrNight.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsLocalSfcErrNight.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsNight.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsNight.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsNight.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsNight.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsStderr.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsStderr.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsStderr.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsStderr.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsTotalDay.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsTotalDay.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsTotalDay.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsTotalDay.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsTotalNight.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsTotalNight.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsTotalNight.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsTotalNight.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsUnCorErrDay.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsUnCorErrDay.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsUnCorErrDay.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsUnCorErrDay.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsUnCorErrNight.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsUnCorErrNight.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsUnCorErrNight.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsUnCorErrNight.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsVarDay.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsVarDay.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsVarDay.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsVarDay.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_tsVarNight.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_tsVarNight.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_tsVarNight.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_tsVarNight.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_txx.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_txx.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_txx.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_txx.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_uajet.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_uajet.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_uajet.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_uajet.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_vegfrac.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_vegfrac.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_vegfrac.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_vegfrac.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_xch4.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_xch4.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_xch4.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_xch4.dat diff --git a/esmvalcore/cmor/tables/custom/CMOR_xco2.dat b/esmvalcore/cmor/tables/cmip5-custom/CMOR_xco2.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_xco2.dat rename to esmvalcore/cmor/tables/cmip5-custom/CMOR_xco2.dat diff --git a/esmvalcore/cmor/tables/cmip6-custom/CMIP6_custom.json b/esmvalcore/cmor/tables/cmip6-custom/CMIP6_custom.json new file mode 100644 index 0000000000..d399838a75 --- /dev/null +++ b/esmvalcore/cmor/tables/cmip6-custom/CMIP6_custom.json @@ -0,0 +1,1649 @@ +{ + "Header": { + "generic_levels": "olevel", + "table_id": "Table custom" + }, + "variable_entry": { + "MP_BC_tot": { + "comment": "positive mass of BC_tot in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of black carbon (sum of all aerosol modes)", + "modeling_realm": "atmos", + "out_name": "MP_BC_tot", + "standard_name": "", + "units": "kg" + }, + "MP_CFCl3": { + "comment": "positive mass of CFCl3", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of CFCl3 (CFC-11)", + "modeling_realm": "atmos", + "out_name": "MP_CFCl3", + "standard_name": "", + "units": "kg" + }, + "MP_CH4": { + "comment": "positive mass of CH4 in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of CH4", + "modeling_realm": "atmos", + "out_name": "MP_CH4", + "standard_name": "", + "units": "kg" + }, + "MP_CO": { + "comment": "positive mass of CO in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of CO", + "modeling_realm": "atmos", + "out_name": "MP_CO", + "standard_name": "", + "units": "kg" + }, + "MP_CO2": { + "comment": "positive mass of CO2 in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of CO2", + "modeling_realm": "atmos", + "out_name": "MP_CO2", + "standard_name": "", + "units": "kg" + }, + "MP_ClOX": { + "comment": "positive mass of ClOX", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of ClOX", + "modeling_realm": "atmos", + "out_name": "MP_ClOX", + "standard_name": "", + "units": "kg" + }, + "MP_DU_tot": { + "comment": "positive mass of DU_tot in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of mineral dust (sum of all aerosol modes)", + "modeling_realm": "atmos", + "out_name": "MP_DU_tot", + "standard_name": "", + "units": "kg" + }, + "MP_N2O": { + "comment": "positive mass of N2O in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of N2O", + "modeling_realm": "atmos", + "out_name": "MP_N2O", + "standard_name": "", + "units": "kg" + }, + "MP_NH3": { + "comment": "positive mass of NH3 in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of NH3", + "modeling_realm": "atmos", + "out_name": "MP_NH3", + "standard_name": "", + "units": "kg" + }, + "MP_NO": { + "comment": "positive mass of NO in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of NO", + "modeling_realm": "atmos", + "out_name": "MP_NO", + "standard_name": "", + "units": "kg" + }, + "MP_NO2": { + "comment": "positive mass of NO2 in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of NO2", + "modeling_realm": "atmos", + "out_name": "MP_NO2", + "standard_name": "", + "units": "kg" + }, + "MP_NOX": { + "comment": "positive mass of NOX in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of NOX (NO+NO2)", + "modeling_realm": "atmos", + "out_name": "MP_NOX", + "standard_name": "", + "units": "kg" + }, + "MP_O3": { + "comment": "positive mass of O3 in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of O3", + "modeling_realm": "atmos", + "out_name": "MP_O3", + "standard_name": "", + "units": "kg" + }, + "MP_OH": { + "comment": "positive mass of OH in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of OH", + "modeling_realm": "atmos", + "out_name": "MP_OH", + "standard_name": "", + "units": "kg" + }, + "MP_S": { + "comment": "positive mass of S in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of S", + "modeling_realm": "atmos", + "out_name": "MP_S", + "standard_name": "", + "units": "kg" + }, + "MP_SO2": { + "comment": "positive mass of SO2 in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of SO2", + "modeling_realm": "atmos", + "out_name": "MP_SO2", + "standard_name": "", + "units": "kg" + }, + "MP_SO4mm_tot": { + "comment": "positive mass of SO4mm_tot in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of aerosol sulfate (sum of all aerosol modes)", + "modeling_realm": "atmos", + "out_name": "MP_SO4mm_tot", + "standard_name": "", + "units": "kg" + }, + "MP_SS_tot": { + "comment": "positive mass of SS_tot in kg", + "dimensions": "time", + "frequency": "10hr", + "long_name": "total mass of sea salt (sum of all aerosol modes)", + "modeling_realm": "atmos", + "out_name": "MP_SS_tot", + "standard_name": "", + "units": "kg" + }, + "agb": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "Amount of living biomass (organic matter) stored in vegetation above the soil including stem, stump, branches, bark, seeds and foliage, expressed as dry weight", + "dimensions": "longitude latitude time", + "long_name": "Above-Ground Biomass", + "modeling_realm": "land", + "out_name": "agb", + "standard_name": "", + "type": "real", + "units": "kg m-2", + "valid_min": "0.0" + }, + "alb": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "calculated from clear-sky fluxes", + "dimensions": "longitude latitude time", + "long_name": "albedo at the surface", + "modeling_realm": "atmos", + "out_name": "alb", + "standard_name": "", + "type": "real", + "units": "1" + }, + "albDiff": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude time", + "long_name": "Difference in surface albedo for a given vegetation cover transition", + "modeling_realm": "atmos", + "out_name": "albDiff", + "standard_name": "", + "type": "real", + "units": "1" + }, + "albDiffiTr13": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude time", + "long_name": "Difference in Surface Albedo for Vegetation Cover Transition from Forest to Crops and Grasses", + "modeling_realm": "atmos", + "out_name": "albDiffiTr13", + "standard_name": "", + "type": "real", + "units": "1" + }, + "amoc": { + "cell_measures": "area: areacello", + "cell_methods": "time: mean area: where sea", + "comment": "AMOC at the Rapid array (26.5 N)", + "dimensions": "time", + "long_name": "Atlantic Meridional Overturning Circulation", + "modeling_realm": "ocean", + "out_name": "amoc", + "standard_name": "", + "type": "real", + "units": "kg s-1" + }, + "asr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude time", + "long_name": "Absorbed shortwave radiation", + "modeling_realm": "atmos", + "out_name": "asr", + "positive": "down", + "standard_name": "", + "type": "real", + "units": "W m-2" + }, + "awhea": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "global mean net surface heat flux over open water", + "dimensions": "longitude latitude time", + "long_name": "Global Mean Net Surface Heat Flux Over Open Water", + "modeling_realm": "atmos", + "out_name": "awhea", + "standard_name": "", + "type": "real", + "units": "W m-2" + }, + "bdalb": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude time", + "long_name": "Broadband directional albedo at the surface", + "modeling_realm": "land", + "out_name": "bdalb", + "standard_name": "", + "type": "real", + "units": "1" + }, + "bhalb": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude time", + "long_name": "Broadband bihemispherical albedo at the surface", + "modeling_realm": "land", + "out_name": "bhalb", + "standard_name": "", + "type": "real", + "units": "1" + }, + "ch4s": { + "cell_measures": "area: areacella", + "cell_methods": "area: time: mean", + "comment": "As ch4, but only at the surface", + "dimensions": "longitude latitude time", + "long_name": "Atmosphere CH4 surface", + "modeling_realm": "atmos", + "out_name": "ch4s", + "standard_name": "mole_fraction_of_methane_in_air", + "type": "real", + "units": "1e-09" + }, + "chlora": { + "cell_measures": "area: areacello", + "cell_methods": "area: mean where sea time: mean", + "comment": "calculated at the surface (to be compared with satellite measurements)", + "dimensions": "longitude latitude time", + "long_name": "chlorophyll concentration", + "modeling_realm": "ocean", + "out_name": "chlora", + "standard_name": "", + "type": "real", + "units": "kg m-3" + }, + "clhmtisccp": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at the top of the atmosphere (to be compared with satellite measurements)", + "dimensions": "longitude latitude time", + "long_name": "ISCCP High Level Medium-Thickness Cloud Area Fraction", + "modeling_realm": "atmos", + "out_name": "clhmtisccp", + "standard_name": "", + "type": "real", + "units": "%" + }, + "clhtkisccp": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at the top of the atmosphere (to be compared with satellite measurements)", + "dimensions": "longitude latitude time", + "long_name": "ISCCP high level thick cloud area fraction", + "modeling_realm": "atmos", + "out_name": "clhtkisccp", + "standard_name": "", + "type": "real", + "units": "%" + }, + "clisccp": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude plev19 tau time", + "long_name": "ISCCP Cloud Area Fraction", + "modeling_realm": "atmos", + "out_name": "clisccp", + "standard_name": "", + "type": "real", + "units": "%" + }, + "cllmtisccp": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at the top of the atmosphere (to be compared with satellite measurements)", + "dimensions": "longitude latitude time", + "long_name": "ISCCP Low Level Medium-Thickness Cloud Area Fraction", + "modeling_realm": "atmos", + "out_name": "cllmtisccp", + "standard_name": "", + "type": "real", + "units": "%" + }, + "clltkisccp": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at the top of the atmosphere (to be compared with satellite measurements)", + "dimensions": "longitude latitude time", + "long_name": "ISCCP low level thick cloud area fraction", + "modeling_realm": "atmos", + "out_name": "clltkisccp", + "standard_name": "", + "type": "real", + "units": "%" + }, + "clmmtisccp": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at the top of the atmosphere (to be compared with satellite measurements)", + "dimensions": "longitude latitude time", + "long_name": "ISCCP Middle Level Medium-Thickness Cloud Area Fraction", + "modeling_realm": "atmos", + "out_name": "clmmtisccp", + "standard_name": "", + "type": "real", + "units": "%" + }, + "clmtkisccp": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at the top of the atmosphere (to be compared with satellite measurements)", + "dimensions": "longitude latitude time", + "long_name": "ISCCP Middle Level Thick Cloud Area Fraction", + "modeling_realm": "atmos", + "out_name": "clmtkisccp", + "standard_name": "", + "type": "real", + "units": "%" + }, + "cltStderr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "for the whole atmospheric column, as seen from the surface or the top of the atmosphere. Include both large-scale and convective cloud.", + "dimensions": "longitude latitude time", + "long_name": "Total Cloud Fraction Error", + "modeling_realm": "atmos", + "ok_max_mean_abs": "0.01", + "ok_min_mean_abs": "0", + "out_name": "cltStderr", + "standard_name": "", + "type": "real", + "units": "%", + "valid_max": "0.01", + "valid_min": "0" + }, + "co2s": { + "cell_measures": "area: areacella", + "cell_methods": "area: time: mean", + "comment": "As co2, but only at the surface", + "dimensions": "longitude latitude time", + "long_name": "Atmosphere CO2", + "modeling_realm": "atmos", + "out_name": "co2s", + "standard_name": "mole_fraction_of_carbon_dioxide_in_air", + "type": "real", + "units": "1e-06" + }, + "ctotal": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean area: mean", + "dimensions": "longitude latitude time", + "long_name": "Total Carbon Mass in Ecosystem", + "modeling_realm": "land", + "out_name": "ctotal", + "standard_name": "", + "type": "real", + "units": "kg m-2" + }, + "dos": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean area: mean where land", + "comment": "(unitless) degree of soil saturation for comparing mass based models with volumetric observations.", + "dimensions": "longitude latitude time", + "long_name": "Degree of Soil Saturation", + "modeling_realm": "land", + "ok_max_mean_abs": "1", + "ok_min_mean_abs": "0", + "out_name": "dos", + "standard_name": "", + "type": "real", + "units": "m3 m-3", + "valid_max": "2", + "valid_min": "0" + }, + "dosStderr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean area: mean where land", + "dimensions": "longitude latitude time", + "long_name": "Degree of Soil Saturation Error", + "modeling_realm": "land", + "out_name": "dosStderr", + "standard_name": "", + "type": "real", + "units": "m3 m-3", + "valid_max": "1.0", + "valid_min": "0.0" + }, + "dpn2o": { + "cell_measures": "area: areacella", + "cell_methods": "area: time: mean", + "comment": "Positive values correspond to higher partial pressure in sea water than air.", + "dimensions": "longitude latitude time", + "long_name": "Surface Nitrous Oxide (N2O) Partial Pressure Difference between Sea Water and Air", + "modeling_realm": "ocnBgchem", + "out_name": "dpn2o", + "standard_name": "", + "type": "real", + "units": "Pa" + }, + "et": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude time", + "long_name": "Evapotranspiration", + "modeling_realm": "atmos", + "out_name": "et", + "standard_name": "", + "type": "real", + "units": "mm day-1" + }, + "etStderr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "Standard deviation", + "dimensions": "longitude latitude time", + "long_name": "Evapotranspiration Error", + "modeling_realm": "land", + "out_name": "etStderr", + "standard_name": "", + "type": "real", + "units": "mm day-1", + "valid_min": "0" + }, + "fapar": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude time", + "long_name": "Fraction of Absorbed Photosynthetically Active Radiation", + "modeling_realm": "land", + "out_name": "fapar", + "standard_name": "", + "type": "real", + "units": "1" + }, + "gppStderr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "Standard deviation calculated based on median absolute deviation", + "dimensions": "longitude latitude time", + "long_name": "Carbon Mass Flux out of Atmosphere due to Gross Primary Production on Land Error", + "modeling_realm": "land", + "out_name": "gppStderr", + "standard_name": "", + "type": "real", + "units": "kg m-2 s-1", + "valid_min": "0" + }, + "hfns": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude time", + "long_name": "Surface Net Heat Flux", + "modeling_realm": "atmos", + "out_name": "hfns", + "positive": "up", + "standard_name": "", + "type": "real", + "units": "W m-2" + }, + "hurStderr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "The relative humidity with respect to liquid water for T> 0 C, and with respect to ice for T<0 C.", + "dimensions": "longitude latitude plev19 time", + "long_name": "Relative Humidity Error", + "modeling_realm": "atmos", + "out_name": "hurStderr", + "standard_name": "", + "type": "real", + "units": "%" + }, + "husStderr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude plev19 time", + "long_name": "Specific Humidity Error", + "modeling_realm": "atmos", + "ok_max_mean_abs": "0.01041", + "ok_min_mean_abs": "-0.0003539", + "out_name": "husStderr", + "standard_name": "", + "type": "real", + "units": "1", + "valid_max": "0.02841", + "valid_min": "-0.000299" + }, + "iwpStderr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude time", + "long_name": "Condensed Ice Path Error", + "modeling_realm": "atmos", + "ok_max_mean_abs": "1.0", + "ok_min_mean_abs": "0.0", + "out_name": "iwpStderr", + "standard_name": "", + "type": "real", + "units": "kg m-2", + "valid_max": "5.0", + "valid_min": "0.0" + }, + "lapserate": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "Atmospheric lapse rate calculated as -dT/dz in K per km.", + "dimensions": "longitude latitude plev19 time", + "long_name": "Lapse Rate", + "modeling_realm": "atmos", + "out_name": "lapserate", + "standard_name": "", + "type": "real", + "units": "K km-1" + }, + "lvp": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "longitude latitude time", + "long_name": "Latent Heat Release from Precipitation", + "modeling_realm": "atmos", + "out_name": "lvp", + "standard_name": "", + "type": "real", + "units": "W m-2" + }, + "lwcre": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at the top of the atmosphere (to be compared with satellite measurements)", + "dimensions": "longitude latitude time", + "long_name": "TOA Longwave Cloud Radiative Effect", + "modeling_realm": "atmos", + "out_name": "lwcre", + "positive": "down", + "standard_name": "", + "type": "real", + "units": "W m-2" + }, + "lweGrace": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean area: mean where land", + "comment": "liquid water equivalent thickness anomaly", + "dimensions": "longitude latitude time", + "long_name": "Liquid Water Equivalent Thickness Anomaly", + "modeling_realm": "land", + "out_name": "lweGrace", + "standard_name": "", + "type": "real", + "units": "m" + }, + "lwp": { + "cell_measures": "area: areacella", + "cell_methods": "area: time: mean", + "comment": "The total mass of liquid water in cloud per unit area.", + "dimensions": "longitude latitude time", + "long_name": "Liquid Water Path", + "modeling_realm": "aerosol", + "out_name": "lwp", + "standard_name": "atmosphere_mass_content_of_cloud_liquid_water", + "type": "real", + "units": "kg m-2" + }, + "lwpStderr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude time", + "long_name": "Liquid Water Path Error", + "modeling_realm": "atmos", + "ok_max_mean_abs": "1.0", + "ok_min_mean_abs": "0.0", + "out_name": "lwpStderr", + "standard_name": "", + "type": "real", + "units": "kg m-2", + "valid_max": "5.0", + "valid_min": "0.0" + }, + "n2oflux": { + "cell_measures": "area: areacella", + "cell_methods": "area: time: mean", + "comment": "Positive flux is into the atmosphere.", + "dimensions": "longitude latitude time", + "long_name": "Surface Upward Ocean to Atmosphere Mole Flux of Nitrous Oxide (N2O)", + "modeling_realm": "ocnBgchem", + "out_name": "n2oflux", + "positive": "up", + "standard_name": "", + "type": "real", + "units": "mol m-2 s-1" + }, + "n2os": { + "cell_measures": "area: areacella", + "cell_methods": "area: time: mean", + "comment": "As n2o, but only at the surface", + "dimensions": "longitude latitude time", + "long_name": "Mole Fraction of N2O at Surface", + "modeling_realm": "atmos", + "out_name": "n2os", + "standard_name": "mole_fraction_of_nitrous_oxide_in_air", + "type": "real", + "units": "mol mol-1" + }, + "netcre": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at the top of the atmosphere (to be compared with satellite measurements)", + "dimensions": "longitude latitude time", + "long_name": "TOA Net Cloud Radiative Effect", + "modeling_realm": "atmos", + "out_name": "netcre", + "positive": "down", + "standard_name": "", + "type": "real", + "units": "W m-2" + }, + "od550aerStderr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "AOD error from the ambient aerosls (i.e., includes aerosol water). Does not include AOD from stratospheric aerosols if these are prescribed but includes other possible background aerosol types.", + "dimensions": "longitude latitude time", + "long_name": "Ambient Aerosol Optical Thickness at 550 nm Error", + "modeling_realm": "aerosol", + "out_name": "od550aerStderr", + "standard_name": "", + "type": "real", + "units": "1" + }, + "od870aerStderr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "AOD error from the ambient aerosls (i.e., includes aerosol water). Does not include AOD from stratospheric aerosols if these are prescribed but includes other possible background aerosol types.", + "dimensions": "longitude latitude time", + "long_name": "Ambient Aerosol Optical Thickness at 870 nm Error", + "modeling_realm": "aerosol", + "out_name": "od870aerStderr", + "standard_name": "", + "type": "real", + "units": "1" + }, + "ohc": { + "cell_measures": "volume: volcello", + "cell_methods": "time: mean area: where sea", + "comment": "Heat content", + "dimensions": "longitude latitude time olevel", + "generic_levels": "olevel", + "long_name": "Heat content in grid cell", + "modeling_realm": "ocean", + "out_name": "ohc", + "standard_name": "", + "type": "real", + "units": "J" + }, + "prStderr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at surface; includes both liquid and solid phases from all types of clouds (both large-scale and convective)", + "dimensions": "longitude latitude time", + "long_name": "Precipitation Standard Error", + "modeling_realm": "atmos", + "ok_max_mean_abs": "0.001", + "ok_min_mean_abs": "0", + "out_name": "prStderr", + "standard_name": "", + "type": "real", + "units": "kg m-2 s-1", + "valid_max": "0.001", + "valid_min": "0" + }, + "prl": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "large scale precipitation at surface; includes both liquid and solid phases from all types of clouds (both large-scale and convective)", + "dimensions": "longitude latitude time", + "long_name": "Large Scale Precipitation", + "modeling_realm": "atmos", + "out_name": "prl", + "standard_name": "", + "type": "real", + "units": "kg m-2 s-1" + }, + "prodlnox": { + "comment": "Production NOX (NO+NO2) by lightning globally integrated", + "dimensions": "time", + "long_name": "Tendency of atmosphere mass content of NOx from lightning", + "modeling_realm": "atmos", + "out_name": "prodlnox", + "standard_name": "", + "type": "real", + "units": "kg s-1" + }, + "ptype": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "Description of numerical values can be found in GRIB2 - CODE TABLE 4.201", + "dimensions": "longitude latitude time", + "long_name": "Precipitation type", + "modeling_realm": "atmos", + "ok_max_mean_abs": "255", + "ok_min_mean_abs": "0", + "out_name": "ptype", + "standard_name": "", + "type": "real", + "units": "1", + "valid_max": "255", + "valid_min": "0" + }, + "qep": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "evspsbl-pr", + "dimensions": "longitude latitude time", + "long_name": "Net moisture flux into atmosphere", + "modeling_realm": "atmos", + "out_name": "qep", + "positive": "up", + "standard_name": "", + "type": "real", + "units": "kg m-2 s-1" + }, + "rlns": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "difference between downward and upward thermal radiation at the surface of the Earth", + "dimensions": "longitude latitude time", + "long_name": "Surface Net downward Longwave Radiation", + "modeling_realm": "atmos", + "out_name": "rlns", + "positive": "down", + "standard_name": "surface_net_downward_longwave_flux", + "type": "real", + "units": "W m-2" + }, + "rlnst": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "to surface and outer space", + "dimensions": "longitude latitude time", + "long_name": "Net Atmospheric Longwave Cooling", + "modeling_realm": "atmos", + "out_name": "rlnst", + "standard_name": "", + "type": "real", + "units": "W m-2" + }, + "rlnstcs": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "to surface and outer space (For definition see: DeAngelis et al. 2015)", + "dimensions": "longitude latitude time", + "long_name": "Net Atmospheric Longwave Cooling assuming clear sky", + "modeling_realm": "atmos", + "out_name": "rlnstcs", + "positive": "up", + "standard_name": "", + "type": "real", + "units": "W m-2" + }, + "rlntcs": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at the top of the atmosphere", + "dimensions": "longitude latitude time", + "long_name": "TOA Net downward Longwave Radiation assuming clear sky", + "modeling_realm": "atmos", + "out_name": "rlntcs", + "positive": "down", + "standard_name": "", + "type": "real", + "units": "W m-2" + }, + "rluscs": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude time", + "long_name": "Surface Upwelling Clear-Sky Longwave Radiation", + "modeling_realm": "atmos", + "ok_max_mean_abs": "376.3", + "ok_min_mean_abs": "325.6", + "out_name": "rluscs", + "positive": "up", + "standard_name": "", + "type": "real", + "units": "W m-2", + "valid_max": "658", + "valid_min": "43.75" + }, + "rlut": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at the top of the atmosphere (to be compared with satellite measurements)", + "dimensions": "longitude latitude time", + "long_name": "TOA Outgoing Longwave Radiation", + "modeling_realm": "atmos", + "ok_max_mean_abs": "234.4", + "ok_min_mean_abs": "207.4", + "out_name": "rlut", + "positive": "up", + "standard_name": "toa_outgoing_longwave_flux", + "type": "real", + "units": "W m-2", + "valid_max": "383.2", + "valid_min": "67.48" + }, + "rlutcs": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude time", + "long_name": "TOA Outgoing Clear-Sky Longwave Radiation", + "modeling_realm": "atmos", + "ok_max_mean_abs": "260.4", + "ok_min_mean_abs": "228.9", + "out_name": "rlutcs", + "positive": "up", + "standard_name": "toa_outgoing_longwave_flux_assuming_clear_sky", + "type": "real", + "units": "W m-2", + "valid_max": "377.5", + "valid_min": "70.59" + }, + "rsns": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "amount of solar radiation that reaches the surface of the Earth minus the amount reflected by the Earth's surface", + "dimensions": "longitude latitude time", + "long_name": "Surface Net downward Shortwave Radiation", + "modeling_realm": "atmos", + "out_name": "rsns", + "positive": "down", + "standard_name": "surface_net_downward_shortwave_flux", + "type": "real", + "units": "W m-2" + }, + "rsnst": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "(For definition see: DeAngelis et al. 2015)", + "dimensions": "longitude latitude time", + "long_name": "Heating from Shortwave Absorption", + "modeling_realm": "atmos", + "out_name": "rsnst", + "standard_name": "", + "type": "real", + "units": "W m-2" + }, + "rsnstcs": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "(For definition see: DeAngelis et al. 2015)", + "dimensions": "longitude latitude time", + "long_name": "Heating from Shortwave Absorption assuming clear sky", + "modeling_realm": "atmos", + "out_name": "rsnstcs", + "positive": "up", + "standard_name": "", + "type": "real", + "units": "W m-2" + }, + "rsnstcsnorm": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "(For definition see: DeAngelis et al. 2015)", + "dimensions": "longitude latitude time", + "long_name": "Heating from Shortwave Absorption assuming clear sky normalized by incoming solar radiation", + "modeling_realm": "atmos", + "out_name": "rsnstcsnorm", + "standard_name": "", + "type": "real", + "units": "%" + }, + "rsnt": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at the top of the atmosphere (to be compared with satellite measurements)", + "dimensions": "longitude latitude time", + "long_name": "TOA Net downward Shortwave Radiation", + "modeling_realm": "atmos", + "out_name": "rsnt", + "positive": "down", + "standard_name": "", + "type": "real", + "units": "W m-2" + }, + "rsntcs": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at the top of the atmosphere", + "dimensions": "longitude latitude time", + "long_name": "TOA Net downward Shortwave Radiation assuming clear sky", + "modeling_realm": "atmos", + "out_name": "rsntcs", + "positive": "down", + "standard_name": "", + "type": "real", + "units": "W m-2" + }, + "rsut": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at the top of the atmosphere", + "dimensions": "longitude latitude time", + "long_name": "TOA Outgoing Shortwave Radiation", + "modeling_realm": "atmos", + "ok_max_mean_abs": "114.1", + "ok_min_mean_abs": "96.72", + "out_name": "rsut", + "positive": "up", + "standard_name": "toa_outgoing_shortwave_flux", + "type": "real", + "units": "W m-2", + "valid_max": "421.9", + "valid_min": "-0.02689" + }, + "rsutcs": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude time", + "long_name": "TOA Outgoing Clear-Sky Shortwave Radiation", + "modeling_realm": "atmos", + "ok_max_mean_abs": "73.36", + "ok_min_mean_abs": "54.7", + "out_name": "rsutcs", + "positive": "up", + "standard_name": "toa_outgoing_shortwave_flux_assuming_clear_sky", + "type": "real", + "units": "W m-2", + "valid_max": "444", + "valid_min": "0" + }, + "rtnt": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at the top of the atmosphere (to be compared with satellite measurements)", + "dimensions": "longitude latitude time", + "long_name": "TOA Net downward Total Radiation", + "modeling_realm": "atmos", + "out_name": "rtnt", + "positive": "down", + "standard_name": "", + "type": "real", + "units": "W m-2" + }, + "rx1day": { + "cell_measures": "area: areacella", + "cell_methods": "time: sum within days time: maximum over days", + "comment": "ETCCDI (Extreme climate change index) annual/monthly maximum 1-day precipitation", + "dimensions": "longitude latitude time", + "frequency": "yr", + "long_name": "Annual/monthly maximum 1-day precipitation", + "modeling_realm": "ground", + "out_name": "rx1day", + "standard_name": "", + "type": "real", + "units": "mm" + }, + "rx5day": { + "cell_measures": "area: areacella", + "cell_methods": "time: sum within days time: maximum over days", + "comment": "ETCCDI (Extreme climate change index) annual/monthly maximum 5-day precipitation", + "dimensions": "longitude latitude time", + "frequency": "yr", + "long_name": "Annual/monthly maximum 5-day precipitation", + "modeling_realm": "ground", + "out_name": "rx5day", + "standard_name": "", + "type": "real", + "units": "mm" + }, + "siextent": { + "cell_measures": "area: areacello", + "cell_methods": "area: mean where sea time: mean", + "comment": "", + "dimensions": "longitude latitude time", + "long_name": "Sea Ice Extent", + "modeling_realm": "seaIce", + "out_name": "siextent", + "standard_name": "", + "type": "real", + "units": "1", + "valid_max": "", + "valid_min": "" + }, + "sispeed": { + "cell_measures": "area: areacella", + "cell_methods": "area: time: mean where sea_ice (comment: mask=siconc)", + "comment": "Speed of ice (i.e. mean absolute velocity) to account for back-and-forth movement of the ice", + "dimensions": "longitude latitude time", + "long_name": "Sea-ice speed", + "modeling_realm": "seaIce", + "out_name": "sispeed", + "standard_name": "sea_ice_speed", + "type": "real", + "units": "m s-1", + "valid_max": "", + "valid_min": "" + }, + "sithick": { + "cell_measures": "area: areacella", + "cell_methods": "area: time: mean where sea_ice (comment: mask=siconc)", + "comment": "Actual (floe) thickness of sea ice (NOT volume divided by grid area as was done in CMIP5)", + "dimensions": "longitude latitude time", + "long_name": "Sea Ice Thickness", + "modeling_realm": "seaIce", + "out_name": "sithick", + "standard_name": "sea_ice_thickness", + "type": "real", + "units": "m", + "valid_max": "", + "valid_min": "" + }, + "sm": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean area: mean where land", + "comment": "the volume of water in all phases in a thin surface soil layer.", + "dimensions": "longitude latitude time", + "long_name": "Volumetric Moisture in Upper Portion of Soil Column", + "modeling_realm": "land", + "ok_max_mean_abs": "1", + "ok_min_mean_abs": "0", + "out_name": "sm", + "standard_name": "", + "type": "real", + "units": "m3 m-3", + "valid_max": "1", + "valid_min": "0" + }, + "sm1m": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean area: mean where land", + "comment": "the volume of water in all phases in the upper 1 metre of the soil column", + "dimensions": "longitude latitude time", + "long_name": "Volumetric Moisture in Upper 1 Metre of Soil Column", + "modeling_realm": "land", + "ok_max_mean_abs": "1", + "ok_min_mean_abs": "0", + "out_name": "sm", + "standard_name": "", + "type": "real", + "units": "m3 m-3", + "valid_max": "1", + "valid_min": "0" + }, + "smStderr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean area: mean where land", + "comment": "Error of the volume of water in all phases in a thin surface soil layer.", + "dimensions": "longitude latitude time", + "long_name": "Volumetric Moisture in Upper Portion of Soil Column Error", + "modeling_realm": "land", + "out_name": "smStderr", + "standard_name": "", + "type": "real", + "units": "m3 m-3", + "valid_max": "1.0", + "valid_min": "0.0" + }, + "soz": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "stratospheric ozone column calculated at 0 degrees C and 1 bar, such that 1m = 1e5 DU. Here, the stratosphere is defined as the region where O3 mole fraction >= 125 ppb.", + "dimensions": "longitude latitude time", + "long_name": "Stratospheric Ozone Column (O3 mole fraction >= 125 ppb)", + "modeling_realm": "atmos", + "out_name": "soz", + "standard_name": "equivalent_thickness_at_stp_of_atmosphere_ozone_content", + "type": "real", + "units": "m", + "valid_max": "5000.0", + "valid_min": "0.0" + }, + "swcre": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "at the top of the atmosphere (to be compared with satellite measurements)", + "dimensions": "longitude latitude time", + "long_name": "TOA Shortwave Cloud Radiative Effect", + "modeling_realm": "atmos", + "out_name": "swcre", + "positive": "down", + "standard_name": "", + "type": "real", + "units": "W m-2" + }, + "tasConf5": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "time", + "long_name": "Near-Surface Air Temperature Uncertainty Range", + "modeling_realm": "atmos", + "ok_max_mean_abs": "20.", + "ok_min_mean_abs": "0", + "out_name": "tasConf5", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "20.", + "valid_min": "0" + }, + "tasConf95": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "time", + "long_name": "Near-Surface Air Temperature Uncertainty Range", + "modeling_realm": "atmos", + "ok_max_mean_abs": "20.", + "ok_min_mean_abs": "0", + "out_name": "tasConf95", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "20.", + "valid_min": "0" + }, + "tasa": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude time", + "long_name": "Near-Surface Air Temperature Anomaly", + "modeling_realm": "atmos", + "ok_max_mean_abs": "20.", + "ok_min_mean_abs": "-20", + "out_name": "tasa", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "20.0", + "valid_min": "-20.0" + }, + "tasaga": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "time", + "long_name": "Global-mean Near-Surface Air Temperature Anomaly", + "modeling_realm": "atmos", + "ok_max_mean_abs": "20.", + "ok_min_mean_abs": "-20", + "out_name": "tasaga", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "20.0", + "valid_min": "-20.0" + }, + "tcw": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "longitude latitude plev19 time", + "long_name": "Mass Fraction of Cloud Total Water (liquid + ice)", + "modeling_realm": "atmos", + "out_name": "tcw", + "standard_name": "", + "type": "real", + "units": "kg kg-1" + }, + "tnn": { + "cell_measures": "area: areacella", + "cell_methods": "time: minimum", + "comment": "ETCCDI (Extreme climate change index) annual/monthly minimum value of daily minimum temperature", + "dimensions": "longitude latitude time", + "frequency": "yr", + "long_name": "Annual/monthly minimum value of daily minimum temperature", + "modeling_realm": "ground", + "out_name": "tnn", + "standard_name": "", + "type": "real", + "units": "degrees_C" + }, + "tosStderr": { + "cell_measures": "area: areacello", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "longitude latitude time", + "long_name": "Sea Surface Temperature Error", + "modeling_realm": "ocean", + "ok_max_mean_abs": "", + "ok_min_mean_abs": "0", + "out_name": "tosStderr", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "", + "valid_min": "0" + }, + "toz": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "Total ozone column calculated at 0 degrees C and 1 bar, such that 1m = 1e5 DU.", + "dimensions": "longitude latitude time", + "long_name": "Total Ozone Column", + "modeling_realm": "atmos", + "out_name": "toz", + "standard_name": "equivalent_thickness_at_stp_of_atmosphere_ozone_content", + "type": "real", + "units": "m", + "valid_max": "5000.0", + "valid_min": "0.0" + }, + "tozStderr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "Total ozone column error calculated at 0 degrees C and 1 bar, such that 1m = 1e5 DU.", + "dimensions": "longitude latitude time", + "long_name": "Total Ozone Column Error", + "modeling_realm": "atmos", + "out_name": "tozStderr", + "standard_name": "equivalent_thickness_at_stp_of_atmosphere_ozone_content", + "type": "real", + "units": "m", + "valid_max": "5000.0", + "valid_min": "0.0" + }, + "tro3prof": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "latitude plev19 time", + "long_name": "Ozone Volume Mixing Ratio", + "modeling_realm": "atmos", + "out_name": "tro3prof", + "standard_name": "", + "type": "real", + "units": "1e-9", + "valid_max": "1.0", + "valid_min": "0.0" + }, + "tro3profStderr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "latitude plev19 time", + "long_name": "Ozone Volume Mixing Ratio Error", + "modeling_realm": "atmos", + "out_name": "tro3profStderr", + "standard_name": "", + "type": "real", + "units": "1e-9", + "valid_max": "1.0", + "valid_min": "0.0" + }, + "troz": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "tropospheric ozone column calculated at 0 degrees C and 1 bar, such that 1m = 1e5 DU. Here, the troposphere is defined as the region where O3 mole fraction < 125 ppb.", + "dimensions": "longitude latitude time", + "long_name": "Tropospheric Ozone Column (O3 mole fraction < 125 ppb)", + "modeling_realm": "atmos", + "out_name": "troz", + "standard_name": "equivalent_thickness_at_stp_of_atmosphere_ozone_content", + "type": "real", + "units": "m", + "valid_max": "5000.0", + "valid_min": "0.0" + }, + "tsDay": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time latitude longitude", + "long_name": "surface temperture daytime", + "modeling_realm": "atmos", + "out_name": "tsDay", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "350", + "valid_min": "190", + "var_name": "tsDay" + }, + "tsLCDay": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time latitude longitude", + "long_name": "land cover class daytime", + "modeling_realm": "atmos", + "out_name": "tsLCDay", + "standard_name": "", + "type": "float", + "units": "", + "valid_max": "255", + "valid_min": "0", + "var_name": "tsLCDay" + }, + "tsLCNight": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time latitude longitude", + "long_name": "land cover class night time", + "modeling_realm": "atmos", + "out_name": "tsLCNight", + "standard_name": "", + "type": "float", + "units": "", + "valid_max": "255", + "valid_min": "0", + "var_name": "tsLCNight" + }, + "tsLSSysErrDay": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time", + "long_name": "uncertainty from large-scale systematic errors daytime", + "modeling_realm": "atmos", + "out_name": "tsLSSysErrDay", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "10", + "valid_min": "0", + "var_name": "tsLSSysErrDay" + }, + "tsLSSysErrNight": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time", + "long_name": "uncertainty from large-scale systematic errors night time", + "modeling_realm": "atmos", + "out_name": "tsLSSysErrNight", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "10", + "valid_min": "0", + "var_name": "tsLSSysErrNight" + }, + "tsLocalAtmErrDay": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time latitude longitude", + "long_name": "uncertainty from locally correlated errors on atmospheric scales daytime", + "modeling_realm": "atmos", + "out_name": "tsLocalAtmErrDay", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "10", + "valid_min": "0", + "var_name": "tsLocalAtmErrDay" + }, + "tsLocalAtmErrNight": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time latitude longitude", + "long_name": "uncertainty from locally correlated errors on atmospheric scales night time", + "modeling_realm": "atmos", + "out_name": "tsLocalAtmErrNight", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "10", + "valid_min": "0", + "var_name": "tsLocalAtmErrNight" + }, + "tsLocalSfcErrDay": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time latitude longitude", + "long_name": "uncertainty from locally correlated errors on surface scales daytime", + "modeling_realm": "atmos", + "out_name": "tsLocalSfcErrDay", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "10", + "valid_min": "0", + "var_name": "tsLocalSfcErrDay" + }, + "tsLocalSfcErrNight": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time latitude longitude", + "long_name": "uncertainty from locally correlated errors on surface scales night time", + "modeling_realm": "atmos", + "out_name": "tsLocalSfcErrNight", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "10", + "valid_min": "0", + "var_name": "tsLocalSfcErrNight" + }, + "tsNight": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time latitude longitude", + "long_name": "surface temperature nighttime", + "modeling_realm": "atmos", + "out_name": "tsNight", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "350", + "valid_min": "190", + "var_name": "tsNight" + }, + "tsStderr": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "\"\"skin\"\" temperature error (i.e., SST for open ocean)", + "dimensions": "longitude latitude time", + "long_name": "Surface Temperature Error", + "modeling_realm": "atmos", + "ok_max_mean_abs": "10", + "ok_min_mean_abs": "0", + "out_name": "tsStderr", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "10", + "valid_min": "0" + }, + "tsTotalDay": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time latitude longitude", + "long_name": "total uncertainty of land surface temperature daytime", + "modeling_realm": "atmos", + "out_name": "tsTotalDay", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "10", + "valid_min": "0", + "var_name": "tsTotalDay" + }, + "tsTotalNight": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time latitude longitude", + "long_name": "total uncertainty of land surface temperature night time", + "modeling_realm": "atmos", + "out_name": "tsTotalNight", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "10", + "valid_min": "0", + "var_name": "tsTotalNight" + }, + "tsUnCorErrDay": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time latitude longitude", + "long_name": "uncertainty from uncorrelated errors daytime", + "modeling_realm": "atmos", + "out_name": "tsUnCorErrDay", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "10", + "valid_min": "0", + "var_name": "tsUnCorErrDay" + }, + "tsUnCorErrNight": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time latitude longitude", + "long_name": "uncertainty from uncorrelated errors night time", + "modeling_realm": "atmos", + "out_name": "tsUnCorErrNight", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "10", + "valid_min": "0", + "var_name": "tsUnCorErrNight" + }, + "tsVarDay": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time latitude longitude", + "long_name": "land surface temperature variance daytime", + "modeling_realm": "atmos", + "out_name": "tsVarDay", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "100", + "valid_min": "0", + "var_name": "tsVarDay" + }, + "tsVarNight": { + "cell_measures": "", + "cell_methods": "time: mean", + "comment": "", + "dimensions": "time latitude longitude", + "long_name": "land surface temperature variance night time", + "modeling_realm": "atmos", + "out_name": "tsVarNight", + "standard_name": "", + "type": "real", + "units": "K", + "valid_max": "100", + "valid_min": "0", + "var_name": "tsVarNight" + }, + "txx": { + "cell_measures": "area: areacella", + "cell_methods": "time: maximum", + "comment": "ETCCDI (Extreme climate change index) annual/monthly maximum value of daily maximum temperature", + "dimensions": "longitude latitude time", + "frequency": "yr", + "long_name": "Annual/monthly maximum value of daily maximum temperature", + "modeling_realm": "ground", + "out_name": "txx", + "standard_name": "", + "type": "real", + "units": "degrees_C" + }, + "uajet": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "dimensions": "time", + "long_name": "Jet position expressed as latitude of maximum meridional wind speed", + "modeling_realm": "atmos", + "out_name": "uajet", + "standard_name": "", + "type": "real", + "units": "degrees" + }, + "vegfrac": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean area: where land", + "comment": "Fraction of entire grid cell that is not covered by bare soil.", + "dimensions": "longitude latitude time", + "long_name": "Vegetation Fraction", + "modeling_realm": "land", + "out_name": "vegfrac", + "standard_name": "", + "type": "real", + "units": "%" + }, + "xch4": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "Satellite retrieved column-average dry-air mole fraction of atmospheric methane (XCH4)", + "dimensions": "longitude latitude time", + "long_name": "Column-average Dry-air Mole Fraction of Atmospheric Methane", + "modeling_realm": "atmos", + "out_name": "xch4", + "standard_name": "", + "type": "real", + "units": "1" + }, + "xco2": { + "cell_measures": "area: areacella", + "cell_methods": "time: mean", + "comment": "Satellite retrieved column-average dry-air mole fraction of atmospheric carbon dioxide (XCO2)", + "dimensions": "longitude latitude time", + "long_name": "Column-average Dry-air Mole Fraction of Atmospheric Carbon Dioxide", + "modeling_realm": "atmos", + "out_name": "xco2", + "standard_name": "", + "type": "real", + "units": "1" + } + } +} diff --git a/esmvalcore/cmor/tables/cmip6-custom/convert-cmip5-to-cmip6.py b/esmvalcore/cmor/tables/cmip6-custom/convert-cmip5-to-cmip6.py new file mode 100644 index 0000000000..4ef96ebaa4 --- /dev/null +++ b/esmvalcore/cmor/tables/cmip6-custom/convert-cmip5-to-cmip6.py @@ -0,0 +1,60 @@ +"""Convert CMIP5-style custom CMOR tables to a CMIP6-style custom table. + +Example usage: `python convert-cmip5-to-cmip6.py esmvalcore/cmor/tables/cmip5-custom/*.dat` +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from collections.abc import Iterable + + +def read(file: Path) -> dict[str, str]: + """Read a CMIP5-style custom CMOR table file into a `dict`.""" + result = {} + for line in file.read_text(encoding="utf-8").split("\n"): + if not line or line.startswith("!"): + continue + key, value = [elem.strip() for elem in line.split(":", 1)] + result[key] = value + return result + + +def translate(files: Iterable[Path]) -> dict[str, Any]: + """Read in CMIP5-style custom CMOR table files and return a CMIP6-style custom table.""" + result = { + "Header": { + "table_id": "Table custom", + "generic_levels": "olevel", + }, + "variable_entry": {}, + } + for file in files: + # Skip the coordinates file and use standard CMIP6 coordinates instead. + if "coordinates" in file.name: + continue + variable = read(file) + variable_entry = variable.pop("variable_entry") + # Remove the "SOURCE" key which has no meaning in CMOR tables. + variable.pop("SOURCE", None) + # Some files are missing `out_name`, assume it is the same as the entry. + if "out_name" not in variable: + variable["out_name"] = variable_entry + # Use a CMIP6 pressure levels coordinate. + if "plevs" in variable["dimensions"]: + variable["dimensions"] = variable["dimensions"].replace( + "plevs", + "plev19", + ) + result["variable_entry"][variable_entry] = variable + return result + + +if __name__ == "__main__": + table_files = [Path(p) for p in sys.argv[1:]] + print(json.dumps(translate(table_files), indent=4, sort_keys=True)) # noqa: T201 diff --git a/esmvalcore/cmor/tables/cmip7/Tables/CMIP7_aerosol.json b/esmvalcore/cmor/tables/cmip7/tables/CMIP7_aerosol.json similarity index 100% rename from esmvalcore/cmor/tables/cmip7/Tables/CMIP7_aerosol.json rename to esmvalcore/cmor/tables/cmip7/tables/CMIP7_aerosol.json diff --git a/esmvalcore/cmor/tables/cmip7/Tables/CMIP7_atmos.json b/esmvalcore/cmor/tables/cmip7/tables/CMIP7_atmos.json similarity index 100% rename from esmvalcore/cmor/tables/cmip7/Tables/CMIP7_atmos.json rename to esmvalcore/cmor/tables/cmip7/tables/CMIP7_atmos.json diff --git a/esmvalcore/cmor/tables/cmip7/Tables/CMIP7_atmosChem.json b/esmvalcore/cmor/tables/cmip7/tables/CMIP7_atmosChem.json similarity index 100% rename from esmvalcore/cmor/tables/cmip7/Tables/CMIP7_atmosChem.json rename to esmvalcore/cmor/tables/cmip7/tables/CMIP7_atmosChem.json diff --git a/esmvalcore/cmor/tables/cmip7/Tables/CMIP7_cell_measures.json b/esmvalcore/cmor/tables/cmip7/tables/CMIP7_cell_measures.json similarity index 100% rename from esmvalcore/cmor/tables/cmip7/Tables/CMIP7_cell_measures.json rename to esmvalcore/cmor/tables/cmip7/tables/CMIP7_cell_measures.json diff --git a/esmvalcore/cmor/tables/cmip7/Tables/CMIP7_coordinate.json b/esmvalcore/cmor/tables/cmip7/tables/CMIP7_coordinate.json similarity index 100% rename from esmvalcore/cmor/tables/cmip7/Tables/CMIP7_coordinate.json rename to esmvalcore/cmor/tables/cmip7/tables/CMIP7_coordinate.json diff --git a/esmvalcore/cmor/tables/cmip7/Tables/CMIP7_formula_terms.json b/esmvalcore/cmor/tables/cmip7/tables/CMIP7_formula_terms.json similarity index 100% rename from esmvalcore/cmor/tables/cmip7/Tables/CMIP7_formula_terms.json rename to esmvalcore/cmor/tables/cmip7/tables/CMIP7_formula_terms.json diff --git a/esmvalcore/cmor/tables/cmip7/Tables/CMIP7_grids.json b/esmvalcore/cmor/tables/cmip7/tables/CMIP7_grids.json similarity index 100% rename from esmvalcore/cmor/tables/cmip7/Tables/CMIP7_grids.json rename to esmvalcore/cmor/tables/cmip7/tables/CMIP7_grids.json diff --git a/esmvalcore/cmor/tables/cmip7/Tables/CMIP7_land.json b/esmvalcore/cmor/tables/cmip7/tables/CMIP7_land.json similarity index 100% rename from esmvalcore/cmor/tables/cmip7/Tables/CMIP7_land.json rename to esmvalcore/cmor/tables/cmip7/tables/CMIP7_land.json diff --git a/esmvalcore/cmor/tables/cmip7/Tables/CMIP7_landIce.json b/esmvalcore/cmor/tables/cmip7/tables/CMIP7_landIce.json similarity index 100% rename from esmvalcore/cmor/tables/cmip7/Tables/CMIP7_landIce.json rename to esmvalcore/cmor/tables/cmip7/tables/CMIP7_landIce.json diff --git a/esmvalcore/cmor/tables/cmip7/Tables/CMIP7_long_name_overrides.json b/esmvalcore/cmor/tables/cmip7/tables/CMIP7_long_name_overrides.json similarity index 100% rename from esmvalcore/cmor/tables/cmip7/Tables/CMIP7_long_name_overrides.json rename to esmvalcore/cmor/tables/cmip7/tables/CMIP7_long_name_overrides.json diff --git a/esmvalcore/cmor/tables/cmip7/Tables/CMIP7_ocean.json b/esmvalcore/cmor/tables/cmip7/tables/CMIP7_ocean.json similarity index 100% rename from esmvalcore/cmor/tables/cmip7/Tables/CMIP7_ocean.json rename to esmvalcore/cmor/tables/cmip7/tables/CMIP7_ocean.json diff --git a/esmvalcore/cmor/tables/cmip7/Tables/CMIP7_ocnBgchem.json b/esmvalcore/cmor/tables/cmip7/tables/CMIP7_ocnBgchem.json similarity index 100% rename from esmvalcore/cmor/tables/cmip7/Tables/CMIP7_ocnBgchem.json rename to esmvalcore/cmor/tables/cmip7/tables/CMIP7_ocnBgchem.json diff --git a/esmvalcore/cmor/tables/cmip7/Tables/CMIP7_seaIce.json b/esmvalcore/cmor/tables/cmip7/tables/CMIP7_seaIce.json similarity index 100% rename from esmvalcore/cmor/tables/cmip7/Tables/CMIP7_seaIce.json rename to esmvalcore/cmor/tables/cmip7/tables/CMIP7_seaIce.json diff --git a/esmvalcore/cmor/tables/cordex-custom/coordinates b/esmvalcore/cmor/tables/cordex-custom/coordinates new file mode 100644 index 0000000000..b2140ede8a --- /dev/null +++ b/esmvalcore/cmor/tables/cordex-custom/coordinates @@ -0,0 +1,43 @@ +table_id: Table custom + +!============ +axis_entry: plevs +!============ +!---------------------------------- +! Axis attributes: +!---------------------------------- +standard_name: air_pressure +units: Pa +axis: Z ! X, Y, Z, T (default: undeclared) +positive: down +long_name: pressure +!---------------------------------- +! Additional axis information: +!---------------------------------- +out_name: plev +valid_min: 0.0 +valid_max: 110000.0 +stored_direction: decreasing +type: double +must_have_bounds: no +!---------------------------------- +! + +!============ +axis_entry: tau +!============ +!---------------------------------- +! Axis attributes: +!---------------------------------- +standard_name: atmosphere_optical_thickness_due_to_cloud +units: 1 +long_name: cloud optical thickness +!---------------------------------- +! Additional axis information: +!---------------------------------- +out_name: tau +stored_direction: increasing +type: double +must_have_bounds: yes +!---------------------------------- +! diff --git a/esmvalcore/cmor/tables/VERSION b/esmvalcore/cmor/tables/obs4mips/VERSION similarity index 100% rename from esmvalcore/cmor/tables/VERSION rename to esmvalcore/cmor/tables/obs4mips/VERSION diff --git a/esmvalcore/cmor/tables/custom/CMOR_coordinates.dat b/esmvalcore/cmor/tables/old-custom-coordinates/CMOR_coordinates.dat similarity index 100% rename from esmvalcore/cmor/tables/custom/CMOR_coordinates.dat rename to esmvalcore/cmor/tables/old-custom-coordinates/CMOR_coordinates.dat diff --git a/esmvalcore/cmor/tables/old-custom-coordinates/README.md b/esmvalcore/cmor/tables/old-custom-coordinates/README.md new file mode 100644 index 0000000000..251fce6248 --- /dev/null +++ b/esmvalcore/cmor/tables/old-custom-coordinates/README.md @@ -0,0 +1,5 @@ +These are custom coordinate variables that were used with the custom CMIP5 +CMOR tables. They are problematic because they overwrite the standard CMIP5 +coordinates and thereby modify the CMIP5 CMOR table when used together with +that table. They are kept here until config-developer.yml and the associated +`esmvalcore.cmor.table.CustomInfo` class is retired in v2.16.0. diff --git a/esmvalcore/config/__init__.py b/esmvalcore/config/__init__.py index 5d23c6b0e2..333c3e30e1 100644 --- a/esmvalcore/config/__init__.py +++ b/esmvalcore/config/__init__.py @@ -13,10 +13,22 @@ """ -from ._config_object import CFG, Config, Session +import contextlib + +import iris + +from esmvalcore.config._config_object import CFG, Config, Session __all__ = ( "CFG", "Config", "Session", ) + +# Set iris.FUTURE flags +for attr, value in { + "save_split_attrs": True, + "date_microseconds": True, +}.items(): + with contextlib.suppress(AttributeError): + setattr(iris.FUTURE, attr, value) diff --git a/esmvalcore/config/_config.py b/esmvalcore/config/_config.py index 3003c6c0e6..3da50569be 100644 --- a/esmvalcore/config/_config.py +++ b/esmvalcore/config/_config.py @@ -1,9 +1,9 @@ """Functions dealing with config-developer.yml and extra facets.""" +# TODO: remove this module in v2.16.0 from __future__ import annotations import collections.abc -import contextlib import logging import os import warnings @@ -11,10 +11,9 @@ from pathlib import Path from typing import TYPE_CHECKING, Any -import iris import yaml -from esmvalcore.cmor.table import CMOR_TABLES, read_cmor_tables +from esmvalcore.cmor.table import read_cmor_tables from esmvalcore.exceptions import ESMValCoreDeprecationWarning, RecipeError if TYPE_CHECKING: @@ -30,15 +29,6 @@ USER_EXTRA_FACETS = Path.home() / ".esmvaltool" / "extra_facets" -# Set iris.FUTURE flags -for attr, value in { - "save_split_attrs": True, - "date_microseconds": True, -}.items(): - with contextlib.suppress(AttributeError): - setattr(iris.FUTURE, attr, value) - - # TODO: remove in v2.15.0 def _deep_update(dictionary, update): for key, value in update.items(): @@ -94,7 +84,10 @@ def warn_if_old_extra_facets_exist() -> None: ) -def load_config_developer(cfg_file: Path) -> dict: +def load_config_developer( + cfg_file: Path, + set_cmor_tables: bool = True, +) -> dict: """Read the developer's configuration file.""" with open(cfg_file, encoding="utf-8") as file: cfg = yaml.safe_load(file) @@ -122,7 +115,8 @@ def load_config_developer(cfg_file: Path) -> dict: settings["input_dir"][site] = normalized_drs CFG[project] = settings - read_cmor_tables(cfg_file) + if set_cmor_tables: + read_cmor_tables(cfg_file) return cfg @@ -134,28 +128,6 @@ def get_project_config(project): raise RecipeError(msg) -def get_institutes(variable): - """Return the institutes given the dataset name in CMIP6.""" - dataset = variable["dataset"] - project = variable["project"] - try: - return CMOR_TABLES[project].institutes[dataset] - except (KeyError, AttributeError): - return [] - - -def get_activity(variable): - """Return the activity given the experiment name in CMIP6.""" - project = variable["project"] - try: - exp = variable["exp"] - if isinstance(exp, list): - return [CMOR_TABLES[project].activities[value][0] for value in exp] - return CMOR_TABLES[project].activities[exp][0] - except (KeyError, AttributeError): - return None - - def get_ignored_warnings(project: FacetValue, step: str) -> None | list: """Get ignored warnings for a given preprocessing step.""" if project not in CFG: diff --git a/esmvalcore/config/_config_validators.py b/esmvalcore/config/_config_validators.py index 272f6f56e0..d2a39c8bc8 100644 --- a/esmvalcore/config/_config_validators.py +++ b/esmvalcore/config/_config_validators.py @@ -7,12 +7,12 @@ import warnings from collections.abc import Iterable from functools import lru_cache, partial -from importlib.resources import files as importlib_files from pathlib import Path from typing import TYPE_CHECKING, Any, Literal from packaging import version +import esmvalcore.cmor.table from esmvalcore import __version__ as current_version from esmvalcore.cmor.check import CheckLevels from esmvalcore.config._config import TASKSEP, load_config_developer @@ -285,9 +285,10 @@ def validate_drs(value): def validate_config_developer(value): """Validate and load config developer path.""" path = validate_path_or_none(value) - if path is None: - path = importlib_files("esmvalcore") / "config-developer.yml" - load_config_developer(path) + if path is not None: + # This has the side-effect of updating `esmvalcore.config._config.CFG` + # and `esmvalcore.cmor.tables.CMOR_TABLES`. + load_config_developer(path) return path @@ -373,17 +374,44 @@ def validate_extra_facets_dir(value): return validate_pathlist(value) +def validate_cmor_tables(value: dict) -> None: + """Validate the CMOR table configuration.""" + # This has the side-effect of updating `esmvalcore.cmor.tables.CMOR_TABLES`. + # + # Relying on global state is not nice, preferably we should get rid of any + # global state except the defaults for starting a new session in + # `esmvalcore.config.CFG`. https://github.com/esMValGroup/esMValCore/issues/2954 + # + # Having side effects when updating an `esmvalcore.config.Session` object + # that changes global state of the `esmvalcore` package is nasty and should + # preferably be avoided. This would require passing around session objects + # instead of relying on global state (e.g. esmvalcore.config.CFG, + # esmvalcore.cmor.tables.CMOR_TABLES). + esmvalcore.cmor.table.CMOR_TABLES = { + project: esmvalcore.cmor.table.get_tables( + session={"projects": value}, # type: ignore[arg-type] + project=project, + ) + for project in value + } + + def validate_projects( value: dict, ) -> dict[str, dict[str, Any]]: """Validate projects mapping.""" mapping = validate_dict(value) options_for_project: dict[str, Callable[[Any], Any]] = { + "cmor_table": validate_dict, "data": validate_dict, # TODO: try to create data sources here "extra_facets": validate_dict, "preprocessor_filename_template": validate_string, } for project, project_config in mapping.items(): + if "cmor_table" not in project_config: + project_config["cmor_table"] = { + "type": "esmvalcore.cmor.table.NoInfo", + } for option, val in project_config.items(): if option not in options_for_project: msg = ( @@ -393,6 +421,7 @@ def validate_projects( ) raise ValidationError(msg) from None mapping[project][option] = options_for_project[option](val) + validate_cmor_tables(mapping) return mapping @@ -453,7 +482,9 @@ def _handle_deprecation( f"been deprecated in ESMValCore version {deprecated_version} and is " f"scheduled for removal in version {remove_version}.{more_info}" ) - warnings.warn(deprecation_msg, ESMValCoreDeprecationWarning, stacklevel=2) + # This function is called by the deprecation functions, which are called by + # ValidatedConfig.__setitem__, so the calling site is 4 levels away. + warnings.warn(deprecation_msg, ESMValCoreDeprecationWarning, stacklevel=4) # TODO: remove in v2.15.0 @@ -611,6 +642,30 @@ def deprecate_search_esgf( ) +def deprecate_config_developer_file( + validated_config: ValidatedConfig, # noqa: ARG001 + value: str | Path, # noqa: ARG001 + validated_value: str | Path, # noqa: ARG001 +) -> None: + """Deprecate ``config_developer_file`` option. + + Parameters + ---------- + validated_config: + ``ValidatedConfig`` instance which will be modified in place. + value: + Raw input value for ``config_file`` option. + validated_value: + Validated value for ``config_file`` option. + + """ + more_info = ( + " Please configure data sources, cmor tables, and preprocessor " + "filename templates under `projects` instead." + ) + _handle_deprecation("config_developer_file", "2.14.0", "2.16.0", more_info) + + # Example usage: see removed files in # https://github.com/ESMValGroup/ESMValCore/pull/2213 _deprecators: dict[str, Callable] = { @@ -619,6 +674,7 @@ def deprecate_search_esgf( "rootpath": deprecate_rootpath, # TODO: remove in v2.16.0 "download_dir": deprecate_download_dir, # TODO: remove in v2.16.0 "search_esgf": deprecate_search_esgf, # TODO: remove in v2.16.0 + "config_developer_file": deprecate_config_developer_file, # TODO: remove in v2.16.0 } @@ -627,4 +683,5 @@ def deprecate_search_esgf( # https://github.com/ESMValGroup/ESMValCore/pull/2213 _deprecated_options_defaults: dict[str, Any] = { "extra_facets_dir": [], # TODO: remove in v2.15.0 + "download_dir": "~/climate_data", # TODO: remove in v2.16.0 } diff --git a/esmvalcore/config/_validated_config.py b/esmvalcore/config/_validated_config.py index 505434c419..c7124b718a 100644 --- a/esmvalcore/config/_validated_config.py +++ b/esmvalcore/config/_validated_config.py @@ -4,7 +4,7 @@ import pprint import warnings -from collections.abc import MutableMapping +from collections.abc import Mapping, MutableMapping from contextlib import contextmanager from copy import deepcopy from typing import TYPE_CHECKING, Any, ClassVar @@ -17,7 +17,7 @@ from ._config_validators import ValidationError if TYPE_CHECKING: - from collections.abc import Callable, Generator, Mapping + from collections.abc import Callable, Generator # The code for this class was take from matplotlib (v3.3) and modified to @@ -81,6 +81,42 @@ def __setitem__(self, key, val): self._mapping[key] = cval + def update(self, *args, **kwargs): # noqa: C901 + """Update configuration with key/value pairs from mapping or kwargs.""" + # This ensures the config_developer_file is always set last when doing + # an update, so the esmvalcore.cmor.table.CMOR_TABLES variable + # is populated with the values from config_developer_file, which is + # needed for backward compatibility. + # TODO: remove in v2.16.0. + config_developer_file_not_set = object() + config_developer_file = config_developer_file_not_set + if args: + if len(args) > 1: + msg = ( + f"Expected at most 1 positional argument, got {len(args)}" + ) + raise TypeError(msg) + other = args[0] + if isinstance(other, Mapping): + for key in other: + if key == "config_developer_file": + config_developer_file = other[key] + continue + self[key] = other[key] + else: + for key, value in other: + if key == "config_developer_file": + config_developer_file = value + continue + self[key] = value + for key, value in kwargs.items(): + if key == "config_developer_file": + config_developer_file = value + continue + self[key] = value + if config_developer_file is not config_developer_file_not_set: + self["config_developer_file"] = config_developer_file + def __getitem__(self, key): """Return value mapped by key.""" try: diff --git a/esmvalcore/config/configurations/defaults/cmor_tables.yml b/esmvalcore/config/configurations/defaults/cmor_tables.yml new file mode 100644 index 0000000000..61462f0761 --- /dev/null +++ b/esmvalcore/config/configurations/defaults/cmor_tables.yml @@ -0,0 +1,110 @@ +# CMOR table configuration. +projects: + # Projects hosted on ESGF. + CMIP7: + cmor_table: + type: esmvalcore.cmor.table.CMIP6Info + paths: + - cmip7/tables + - cmip6-custom + CMIP6: + cmor_table: + type: esmvalcore.cmor.table.CMIP6Info + paths: + - cmip6/Tables + - cmip6-custom + CMIP5: + cmor_table: + type: esmvalcore.cmor.table.CMIP5Info + paths: + - cmip5/Tables + - cmip5-custom + CMIP3: + cmor_table: + type: esmvalcore.cmor.table.CMIP3Info + paths: + - cmip3/Tables + - cmip3-custom + CORDEX: + cmor_table: + type: esmvalcore.cmor.table.CMIP5Info + paths: + - cordex/Tables + - cordex-custom + - cmip5-custom + obs4MIPs: + cmor_table: + type: esmvalcore.cmor.table.Obs4MIPsInfo + # Set `strict: false` because the tables are incomplete. + strict: false + paths: + - obs4mips/Tables + - cmip6-custom + ana4MIPs: + # CMOR tables are available at https://github.com/PCMDI/ana4MIPs-cmor-tables/ + # but it is not clear how well these match the data. + cmor_table: + type: esmvalcore.cmor.table.CMIP5Info + paths: + - cmip5/Tables + - cmip5-custom + strict: false + # Observational and reanalysis data that can be read in its native format by ESMValCore. + native6: + cmor_table: + type: esmvalcore.cmor.table.CMIP6Info + paths: + - cmip6/Tables + - cmip6-custom + strict: false + # Data from selected climate models that can be read in its native format by ESMValCore. + ACCESS: + cmor_table: + type: esmvalcore.cmor.table.CMIP6Info + paths: + - cmip6/Tables + - cmip6-custom + strict: false + CESM: + cmor_table: + type: esmvalcore.cmor.table.CMIP6Info + paths: + - cmip6/Tables + - cmip6-custom + strict: false + EMAC: + cmor_table: + type: esmvalcore.cmor.table.CMIP6Info + paths: + - cmip6/Tables + - cmip6-custom + strict: false + ICON: + cmor_table: + type: esmvalcore.cmor.table.CMIP6Info + paths: + - cmip6/Tables + - cmip6-custom + strict: false + IPSLCM: + cmor_table: + type: esmvalcore.cmor.table.CMIP6Info + paths: + - cmip6/Tables + - cmip6-custom + strict: false + # Data that has been CMORized by ESMValTool + OBS6: + cmor_table: + type: esmvalcore.cmor.table.CMIP6Info + paths: + - cmip6/Tables + - cmip6-custom + strict: false + OBS: + cmor_table: + type: esmvalcore.cmor.table.CMIP5Info + paths: + - cmip5/Tables + - cmip5-custom + strict: false diff --git a/esmvalcore/config/configurations/defaults/config-user.yml b/esmvalcore/config/configurations/defaults/config-user.yml index a8b959a70e..bfc7a157c5 100644 --- a/esmvalcore/config/configurations/defaults/config-user.yml +++ b/esmvalcore/config/configurations/defaults/config-user.yml @@ -54,11 +54,6 @@ compress_netcdf: false # step. These files are numbered according to the preprocessing order. save_intermediary_cubes: false -# Path to custom ``config-developer.yml`` file -# This can be used to customise project configurations. See -# ``config-developer.yml`` for an example. Set to ``null`` to use the default. -config_developer_file: null - # Use a profiling tool for the diagnostic run --- [false]/true # A profiler tells you which functions in your code take most time to run. # Only available for Python diagnostics. diff --git a/esmvalcore/dataset.py b/esmvalcore/dataset.py index d2cefb084b..c2ef55336d 100644 --- a/esmvalcore/dataset.py +++ b/esmvalcore/dataset.py @@ -19,16 +19,14 @@ from esmvalcore._recipe import check from esmvalcore._recipe.from_datasets import datasets_to_recipe from esmvalcore.cmor.table import ( + NoInfo, _get_branding_suffixes, _get_mips, _update_cmor_facets, + get_tables, ) from esmvalcore.config import CFG -from esmvalcore.config._config import ( - get_activity, - get_institutes, - load_extra_facets, -) +from esmvalcore.config._config import load_extra_facets from esmvalcore.config._data_sources import _get_data_sources from esmvalcore.exceptions import InputFilesNotFound, RecipeError from esmvalcore.io.local import _dates_to_timerange @@ -726,14 +724,6 @@ def _get_extra_facets(self) -> dict[str, Any]: def _augment_facets(self) -> None: extra_facets = self._get_extra_facets() _augment(self.facets, extra_facets) - if "institute" not in self.facets: - institute = get_institutes(self.facets) - if institute: - self.facets["institute"] = institute - if "activity" not in self.facets: - activity = get_activity(self.facets) - if activity: - self.facets["activity"] = activity _update_cmor_facets(self.facets) if self.facets.get("frequency") == "fx": self.facets.pop("timerange", None) @@ -841,28 +831,39 @@ def _load(self) -> Cube: self.session._fixed_file_dir, # noqa: SLF001 self._get_joined_summary_facets("_", join_lists=True) + "_", ) + cmor_tables_available = not isinstance( + get_tables( + session=self.session, + project=self.facets["project"], # type: ignore[arg-type] + ), + NoInfo, + ) settings: dict[str, dict[str, Any]] = {} - settings["fix_file"] = { - "output_dir": fix_dir_prefix, - "add_unique_suffix": True, - "session": self.session, - **self.facets, - } + if cmor_tables_available: + settings["fix_file"] = { + "output_dir": fix_dir_prefix, + "add_unique_suffix": True, + "session": self.session, + **self.facets, + } settings["load"] = {} - settings["fix_metadata"] = { - "session": self.session, - **self.facets, - } + if cmor_tables_available: + settings["fix_metadata"] = { + "session": self.session, + **self.facets, + } settings["concatenate"] = {"check_level": self.session["check_level"]} - settings["cmor_check_metadata"] = { - "check_level": self.session["check_level"], - "cmor_table": self.facets["project"], - "mip": self.facets["mip"], - "frequency": self.facets["frequency"], - "short_name": self.facets["short_name"], - "branding_suffix": self.facets.get("branding_suffix"), - } + + if cmor_tables_available: + settings["cmor_check_metadata"] = { + "check_level": self.session["check_level"], + "cmor_table": self.facets["project"], + "mip": self.facets["mip"], + "frequency": self.facets["frequency"], + "short_name": self.facets["short_name"], + "branding_suffix": self.facets.get("branding_suffix"), + } if "timerange" in self.facets: settings["clip_timerange"] = { "timerange": self.facets["timerange"], @@ -871,14 +872,15 @@ def _load(self) -> Cube: "session": self.session, **self.facets, } - settings["cmor_check_data"] = { - "check_level": self.session["check_level"], - "cmor_table": self.facets["project"], - "mip": self.facets["mip"], - "frequency": self.facets["frequency"], - "short_name": self.facets["short_name"], - "branding_suffix": self.facets.get("branding_suffix"), - } + if cmor_tables_available: + settings["cmor_check_data"] = { + "check_level": self.session["check_level"], + "cmor_table": self.facets["project"], + "mip": self.facets["mip"], + "frequency": self.facets["frequency"], + "short_name": self.facets["short_name"], + "branding_suffix": self.facets.get("branding_suffix"), + } result: Sequence[PreprocessorItem] = self.files for step, kwargs in settings.items(): diff --git a/esmvalcore/io/__init__.py b/esmvalcore/io/__init__.py index e115c462f2..5a936d47e0 100644 --- a/esmvalcore/io/__init__.py +++ b/esmvalcore/io/__init__.py @@ -67,8 +67,8 @@ def load_data_sources( If no ``priority`` is configured for a data source, the default priority of 1 is used. - Arguments - --------- + Parameters + ---------- session: The configuration. project: diff --git a/esmvalcore/io/local.py b/esmvalcore/io/local.py index 56d0da90da..aad0d7dccf 100644 --- a/esmvalcore/io/local.py +++ b/esmvalcore/io/local.py @@ -600,6 +600,9 @@ def find_data(self, **facets: FacetValue) -> list[LocalFile]: f" within the requested timerange {facets['timerange']}" ) + if files: + self.debug_info = f"F{self.debug_info[len('No f') :]}" + return files def _path2facets(self, path: Path, add_timerange: bool) -> dict[str, str]: diff --git a/esmvalcore/local.py b/esmvalcore/local.py index 850d32d0ef..97c3822412 100644 --- a/esmvalcore/local.py +++ b/esmvalcore/local.py @@ -15,7 +15,12 @@ from typing import TYPE_CHECKING, Any from esmvalcore.config import CFG -from esmvalcore.config._config import get_ignored_warnings, get_project_config +from esmvalcore.config._config import CFG as CONFIG_DEVELOPER +from esmvalcore.config._config import ( + get_ignored_warnings, + get_project_config, + load_config_developer, +) from esmvalcore.io.local import ( LocalDataSource, LocalFile, @@ -37,6 +42,17 @@ logger = logging.getLogger(__name__) +def _ensure_config_developer_drs() -> Path: + """Ensure that directory structure from config-developer.yml is loaded.""" + config_developer_file = CFG.get("config_developer_file") + if not config_developer_file: + config_developer_file = Path(__file__).parent / "config-developer.yml" + if not CONFIG_DEVELOPER: + # Load the config-developer.yml file, but do not update the CMOR tables. + load_config_developer(config_developer_file, set_cmor_tables=False) + return config_developer_file + + def _select_drs(input_type: str, project: str, structure: str) -> list[str]: """Select the directory structure of input path.""" cfg = get_project_config(project) @@ -61,6 +77,7 @@ def _select_drs(input_type: str, project: str, structure: str) -> list[str]: def _get_data_sources(project: str) -> list[LocalDataSource]: """Get a list of data sources.""" + config_developer_file = _ensure_config_developer_drs() rootpaths = CFG["rootpath"] default_drs = { "CMIP3": "ESGF", @@ -110,7 +127,7 @@ def _get_data_sources(project: str) -> list[LocalDataSource]: "and 'drs' settings and the path templates from '%s'" ), project, - CFG["config_developer_file"], + config_developer_file, ) _LEGACY_DATA_SOURCES_WARNED.add(project) return sources @@ -264,6 +281,7 @@ def find_files( def _get_output_file(variable: dict[str, Any], preproc_dir: Path) -> Path: """Return the full path to the output (preprocessed) file.""" + _ensure_config_developer_drs() project = variable["project"] cfg = get_project_config(project) if project not in _GET_OUTPUT_FILE_WARNED: diff --git a/esmvalcore/preprocessor/_derive/toz.py b/esmvalcore/preprocessor/_derive/toz.py index bf123c2a39..3f44f42684 100644 --- a/esmvalcore/preprocessor/_derive/toz.py +++ b/esmvalcore/preprocessor/_derive/toz.py @@ -4,7 +4,7 @@ import iris from scipy import constants -from esmvalcore.cmor.table import CMOR_TABLES +import esmvalcore.cmor.table from esmvalcore.iris_helpers import ignore_iris_vague_metadata_warnings from esmvalcore.preprocessor._regrid import extract_levels, regrid @@ -48,7 +48,9 @@ def add_longitude_coord(cube): def interpolate_hybrid_plevs(cube): """Interpolate hybrid pressure levels.""" # Use CMIP6's plev19 target levels (in Pa) - target_levels = CMOR_TABLES["CMIP6"].coords["plev19"].requested + target_levels = ( + esmvalcore.cmor.table.CMOR_TABLES["CMIP6"].coords["plev19"].requested + ) cube.coord("air_pressure").convert_units("Pa") return extract_levels( cube, diff --git a/esmvalcore/preprocessor/_other.py b/esmvalcore/preprocessor/_other.py index 972f93acbb..cc48022804 100644 --- a/esmvalcore/preprocessor/_other.py +++ b/esmvalcore/preprocessor/_other.py @@ -43,6 +43,7 @@ def align_metadata( target_project: str, target_mip: str, target_short_name: str, + target_branding_suffix: str | None = None, strict: bool = True, ) -> Cube: """Set cube metadata to entries from a specific target project. @@ -64,6 +65,8 @@ def align_metadata( MIP table from which target metadata is read. target_short_name: Variable short name from which target metadata is read. + target_branding_suffix: + Branding suffix from which target metadata is read. strict: If ``True``, raise an error if desired metadata cannot be read for variable ``target_short_name`` of MIP table ``target_mip`` and project @@ -86,7 +89,12 @@ def align_metadata( cube = cube.copy() try: - var_info = _get_var_info(target_project, target_mip, target_short_name) + var_info = _get_var_info( + target_project, + target_mip, + target_short_name, + branding_suffix=target_branding_suffix, + ) except ValueError as exc: if strict: raise @@ -101,13 +109,28 @@ def align_metadata( return cube -def _get_var_info(project: str, mip: str, short_name: str) -> VariableInfo: +def _get_var_info( + project: str, + mip: str, + short_name: str, + branding_suffix: str | None, +) -> VariableInfo: """Get variable information.""" - var_info = get_var_info(project, mip, short_name) + var_info = get_var_info( + project, + mip, + short_name, + branding_suffix=branding_suffix, + ) if var_info is None: msg = ( - f"Variable '{short_name}' not available for table '{mip}' of " - f"project '{project}'" + f"Variable '{short_name}' " + + ( + f"with branding suffix '{branding_suffix}' " + if branding_suffix + else "" + ) + + f"not available for table '{mip}' of project '{project}'" ) raise ValueError(msg) return var_info diff --git a/esmvalcore/preprocessor/_regrid.py b/esmvalcore/preprocessor/_regrid.py index 486b831e79..461e75002a 100644 --- a/esmvalcore/preprocessor/_regrid.py +++ b/esmvalcore/preprocessor/_regrid.py @@ -28,11 +28,11 @@ from iris.cube import Cube from iris.util import broadcast_to_shape +import esmvalcore.cmor.table from esmvalcore.cmor._fixes.shared import ( add_altitude_from_plev, add_plev_from_altitude, ) -from esmvalcore.cmor.table import CMOR_TABLES from esmvalcore.iris_helpers import has_irregular_grid, has_unstructured_grid from esmvalcore.preprocessor._shared import ( _rechunk_aux_factory_dependencies, @@ -1403,15 +1403,15 @@ def get_cmor_levels(cmor_table: str, coordinate: str) -> list[float]: If the CMOR table is not defined, the coordinate does not specify any levels or the string is badly formatted. """ - if cmor_table not in CMOR_TABLES: + if cmor_table not in esmvalcore.cmor.table.CMOR_TABLES: msg = f"Level definition cmor_table '{cmor_table}' not available" raise ValueError(msg) - if coordinate not in CMOR_TABLES[cmor_table].coords: + if coordinate not in esmvalcore.cmor.table.CMOR_TABLES[cmor_table].coords: msg = f"Coordinate {coordinate} not available for {cmor_table}" raise ValueError(msg) - cmor = CMOR_TABLES[cmor_table].coords[coordinate] + cmor = esmvalcore.cmor.table.CMOR_TABLES[cmor_table].coords[coordinate] if cmor.requested: return [float(level) for level in cmor.requested] diff --git a/tests/integration/cmor/_fixes/native6/test_era5.py b/tests/integration/cmor/_fixes/native6/test_era5.py index b183bd811b..c899857bce 100644 --- a/tests/integration/cmor/_fixes/native6/test_era5.py +++ b/tests/integration/cmor/_fixes/native6/test_era5.py @@ -17,7 +17,8 @@ fix_accumulated_units, ) from esmvalcore.cmor.fix import fix_metadata -from esmvalcore.cmor.table import CMOR_TABLES, get_var_info +from esmvalcore.cmor.table import get_tables +from esmvalcore.config import CFG from esmvalcore.dataset import Dataset from esmvalcore.preprocessor import cmor_check_metadata @@ -26,18 +27,22 @@ f"{datetime.datetime.now().year}" ) +CMOR_TABLES = { + "native6": get_tables(CFG, "native6"), +} + def test_get_evspsbl_fix(): """Test whether the right fixes are gathered for a single variable.""" fix = Fix.get_fixes("native6", "ERA5", "E1hr", "evspsbl") - vardef = get_var_info("native6", "E1hr", "evspsbl") + vardef = CMOR_TABLES["native6"].get_variable("E1hr", "evspsbl") assert fix == [Evspsbl(vardef), AllVars(vardef), GenericFix(vardef)] def test_get_zg_fix(): """Test whether the right fix gets found again, for zg as well.""" fix = Fix.get_fixes("native6", "ERA5", "Amon", "zg") - vardef = get_var_info("native6", "E1hr", "evspsbl") + vardef = CMOR_TABLES["native6"].get_variable("E1hr", "evspsbl") assert fix == [Zg(vardef), AllVars(vardef), GenericFix(vardef)] @@ -674,8 +679,6 @@ def ptype_cmor_e1hr(): ], attributes={"comment": COMMENT}, ) - cube.coord("latitude").long_name = "latitude" - cube.coord("longitude").long_name = "longitude" return CubeList([cube]) @@ -752,8 +755,6 @@ def rlns_cmor_e1hr(): ], attributes={"comment": COMMENT, "positive": "down"}, ) - cube.coord("latitude").long_name = "latitude" # from custom table - cube.coord("longitude").long_name = "longitude" # from custom table return CubeList([cube]) @@ -984,8 +985,6 @@ def rsns_cmor_e1hr(): ], attributes={"comment": COMMENT, "positive": "down"}, ) - cube.coord("latitude").long_name = "latitude" # from custom table - cube.coord("longitude").long_name = "longitude" # from custom table return CubeList([cube]) diff --git a/tests/integration/cmor/test_read_cmor_tables.py b/tests/integration/cmor/test_read_cmor_tables.py index 86c90454f4..e05f7c6637 100644 --- a/tests/integration/cmor/test_read_cmor_tables.py +++ b/tests/integration/cmor/test_read_cmor_tables.py @@ -1,11 +1,25 @@ +from __future__ import annotations + from pathlib import Path from textwrap import dedent +from typing import TYPE_CHECKING import pytest import yaml -from esmvalcore.cmor.table import CMOR_TABLES, read_cmor_tables +import esmvalcore.cmor.table +from esmvalcore.cmor.table import ( + _TABLE_CACHE, + VariableInfo, + clear_table_cache, + get_tables, + read_cmor_tables, +) from esmvalcore.cmor.table import __file__ as root +from esmvalcore.exceptions import InvalidConfigParameter + +if TYPE_CHECKING: + from esmvalcore.config import Session def test_read_cmor_tables_raiser(): @@ -16,40 +30,190 @@ def test_read_cmor_tables_raiser(): assert "cow" in str(exc) -def test_read_cmor_tables(): +def test_read_cmor_tables_from_config_developer(monkeypatch): """Test that the function `read_cmor_tables` loads the tables correctly.""" + monkeypatch.setattr(esmvalcore.cmor.table, "CMOR_TABLES", {}) + read_cmor_tables() table_path = Path(root).parent / "tables" for project in "CMIP5", "CMIP6": - table = CMOR_TABLES[project] - assert ( - Path(table._cmor_folder) == table_path / project.lower() / "Tables" - ) + table = esmvalcore.cmor.table.CMOR_TABLES[project] + assert table.paths == (table_path / project.lower() / "Tables",) assert table.strict is True project = "OBS" - table = CMOR_TABLES[project] - assert Path(table._cmor_folder) == table_path / "cmip5" / "Tables" + table = esmvalcore.cmor.table.CMOR_TABLES[project] + assert table.paths == (table_path / "cmip5" / "Tables",) assert table.strict is False project = "OBS6" - table = CMOR_TABLES[project] - assert Path(table._cmor_folder) == table_path / "cmip6" / "Tables" + table = esmvalcore.cmor.table.CMOR_TABLES[project] + assert table.paths == (table_path / "cmip6" / "Tables",) assert table.strict is False project = "obs4MIPs" - table = CMOR_TABLES[project] - assert Path(table._cmor_folder) == table_path / "obs4mips" / "Tables" + table = esmvalcore.cmor.table.CMOR_TABLES[project] + assert table.paths == (table_path / "obs4mips" / "Tables",) assert table.strict is False project = "custom" - table = CMOR_TABLES[project] - assert Path(table._cmor_folder) == table_path / "custom" - assert table._user_table_folder is None + table = esmvalcore.cmor.table.CMOR_TABLES[project] + assert table.paths == ( + table_path / "old-custom-coordinates", + table_path / "cmip5-custom", + ) assert table.coords assert table.tables["custom"] +@pytest.mark.parametrize( + ( + "project", + "mip", + "short_name", + "branding_suffix", + ), + [ + ("CMIP7", "atmos", "tas", "tavg-h2m-hxy-u"), + ("CMIP7", "Amon", "alb", None), # custom derived variable + ("CMIP6", "Amon", "tas", None), + ("CMIP6", "Amon", "alb", None), # custom derived variable + ("CMIP6", "Amon", "ch4", "Clim"), # table entry != short_name + ("CMIP5", "Amon", "tas", None), + ("CMIP5", "Amon", "alb", None), # custom derived variable + ("CMIP3", "A1", "tas", None), + ("CMIP3", "A1", "lwcre", None), # custom derived variable + ("CORDEX", "mon", "tas", None), + ("CORDEX", "mon", "alb", None), # custom derived variable + ("obs4MIPs", "Amon", "tas", None), + ("obs4MIPs", "Amon", "agb", None), # custom variable + ("obs4MIPs", "Amon", "alb", None), # custom derived variable + ("ana4MIPs", "Amon", "tas", None), + ("native6", "Amon", "tas", None), + ("native6", "Amon", "agb", None), # custom variable + ("native6", "Amon", "alb", None), # custom derived variable + ("ACCESS", "Amon", "tas", None), + ("CESM", "Amon", "tas", None), + ("EMAC", "Amon", "tas", None), + ("ICON", "Amon", "tas", None), + ("IPSLCM", "Amon", "tas", None), + ("OBS6", "Amon", "tas", None), + ("OBS6", "Amon", "agb", None), # custom variable + ("OBS6", "Amon", "alb", None), # custom derived variable + ("OBS", "Amon", "tas", None), + ("OBS", "Amon", "agb", None), # custom variable + ("OBS", "Amon", "alb", None), # custom derived variable + ], +) +def test_get_tables( + session: Session, + project: str, + mip: str, + short_name: str, + branding_suffix: str | None, +) -> None: + info = get_tables(session, project) + assert info.tables + vardef = info.get_variable( + mip, + short_name, + branding_suffix=branding_suffix, + derived=short_name in ("alb", "lwcre"), + ) + assert isinstance(vardef, VariableInfo) + assert vardef.short_name + assert vardef.units + + +def test_get_tables_unknown_project( + session: Session, +) -> None: + with pytest.raises( + ValueError, + match=r"Unknown project 'unknown', please configure it under 'projects'.", + ): + get_tables(session, "unknown") + + +def test_get_tables_no_type( + session: Session, +) -> None: + session["projects"]["test"] = {"cmor_table": {}} + with pytest.raises( + ValueError, + match=( + r"Missing CMOR table 'type' in configuration of project " + r"test. Current configuration is:\n{}\n" + ), + ): + get_tables(session, "test") + + +def test_get_tables_non_existent_table_module( + session: Session, +) -> None: + session["projects"]["test"] = { + "cmor_table": { + "type": "tests.integration.cmor.does_not_exist.DoesNotExist", + }, + } + with pytest.raises( + InvalidConfigParameter, + match=( + r"Failed to import module 'tests.integration.cmor.does_not_exist' " + r"for CMOR table of project 'test'. Please check your configuration. " + ), + ): + get_tables(session, "test") + + +def test_get_tables_non_existent_table_class( + session: Session, +) -> None: + session["projects"]["test"] = { + "cmor_table": { + "type": "tests.integration.cmor.test_read_cmor_tables.NonExistentTable", + }, + } + with pytest.raises( + InvalidConfigParameter, + match=( + r"Class 'NonExistentTable' for reading CMOR table of project 'test' " + r"does not exist in module 'tests.integration.cmor.test_read_cmor_tables'. " + r"Please check your configuration." + ), + ): + get_tables(session, "test") + + +class InvalidTable: + pass + + +def test_get_tables_invalid_type( + session: Session, +) -> None: + session["projects"]["test"] = { + "cmor_table": { + "type": "tests.integration.cmor.test_read_cmor_tables.InvalidTable", + }, + } + with pytest.raises( + TypeError, + match=( + r"`type` should be a subclass `esmvalcore.cmor.table.InfoBase`, " + r"but your configuration for project 'test' contains " + ), + ): + get_tables(session, "test") + + +def test_clear_table_cache(session: Session) -> None: + assert _TABLE_CACHE + clear_table_cache() + assert not _TABLE_CACHE + + CMOR_NEWVAR_ENTRY = dedent( """ !============ @@ -125,8 +289,9 @@ def test_read_cmor_tables(): ) -def test_read_custom_cmor_tables(tmp_path): +def test_read_custom_cmor_tables_config_developer(tmp_path, monkeypatch): """Test reading of custom CMOR tables.""" + monkeypatch.setattr(esmvalcore.cmor.table, "CMOR_TABLES", {}) (tmp_path / "CMOR_newvarfortesting.dat").write_text(CMOR_NEWVAR_ENTRY) (tmp_path / "CMOR_netcre.dat").write_text(CMOR_NETCRE_ENTRY) (tmp_path / "CMOR_coordinates.dat").write_text(CMOR_NEWCOORD_ENTRY) @@ -147,15 +312,16 @@ def test_read_custom_cmor_tables(tmp_path): read_cmor_tables(cfg_file) - assert len(CMOR_TABLES) == 2 - assert "CMIP6" in CMOR_TABLES - assert "custom" in CMOR_TABLES + assert len(esmvalcore.cmor.table.CMOR_TABLES) == 2 + assert "CMIP6" in esmvalcore.cmor.table.CMOR_TABLES + assert "custom" in esmvalcore.cmor.table.CMOR_TABLES - custom_table = CMOR_TABLES["custom"] - assert custom_table._cmor_folder == str( - Path(root).parent / "tables" / "custom", + custom_table = esmvalcore.cmor.table.CMOR_TABLES["custom"] + assert custom_table.paths == ( + Path(root).parent / "tables" / "old-custom-coordinates", + Path(root).parent / "tables" / "cmip5-custom", + tmp_path, ) - assert custom_table._user_table_folder == str(tmp_path) # Make sure that default tables have been read assert "alb" in custom_table.tables["custom"] @@ -169,7 +335,7 @@ def test_read_custom_cmor_tables(tmp_path): assert netcre.units == "K" assert netcre.long_name == "This is New" - cmip6_table = CMOR_TABLES["CMIP6"] + cmip6_table = esmvalcore.cmor.table.CMOR_TABLES["CMIP6"] assert cmip6_table.default is custom_table # Restore default tables diff --git a/tests/integration/cmor/test_table.py b/tests/integration/cmor/test_table.py index 08f4e76795..3e7f43f455 100644 --- a/tests/integration/cmor/test_table.py +++ b/tests/integration/cmor/test_table.py @@ -1,17 +1,23 @@ """Integration tests for the variable_info module.""" +from __future__ import annotations + +import logging import os -import unittest from pathlib import Path import pytest import esmvalcore.cmor +import esmvalcore.cmor.table from esmvalcore.cmor.table import ( CMIP3Info, CMIP5Info, CMIP6Info, CustomInfo, + NoInfo, + Obs4MIPsInfo, + VariableInfo, _get_branding_suffixes, _get_mips, _update_cmor_facets, @@ -24,6 +30,8 @@ def test_update_cmor_facets(): "project": "CMIP6", "mip": "Amon", "short_name": "tas", + "dataset": "CanESM5", + "exp": "historical", } _update_cmor_facets(facets) @@ -32,12 +40,18 @@ def test_update_cmor_facets(): "project": "CMIP6", "mip": "Amon", "short_name": "tas", + "dataset": "CanESM5", "original_short_name": "tas", "standard_name": "air_temperature", "long_name": "Near-Surface Air Temperature", "units": "K", "modeling_realm": ["atmos"], "frequency": "mon", + "activity": "CMIP", + "exp": "historical", + "institute": [ + "CCCma", + ], } assert facets == expected @@ -66,18 +80,16 @@ def test_update_cmor_facets_facet_not_in_table(mocker): assert facets == expected -class TestCMIP6Info(unittest.TestCase): - """Test for the CMIP6 info class.""" - - @classmethod - def setUpClass(cls): - """Set up tests. +class TestCMIP6Info: + """Tests for the CMIP6 info class.""" - We read CMIP6Info once to keep tests times manageable - """ - cls.variables_info = CMIP6Info( - "cmip6", - default=CustomInfo(), + @pytest.fixture + def variables_info(self) -> CMIP6Info: + return CMIP6Info( + paths=[ + Path("cmip6/Tables"), + Path("cmip6-custom"), + ], strict=True, alt_names=[ ["sic", "siconc"], @@ -85,8 +97,17 @@ def setUpClass(cls): ], ) - def setUp(self): - self.variables_info.strict = True + def test_repr(self, variables_info: CMIP6Info) -> None: + builtin_tables_path = Path(esmvalcore.cmor.__file__).parent / "tables" + expected_paths = [ + builtin_tables_path / "cmip6" / "Tables", + builtin_tables_path / "cmip6-custom", + ] + result = repr(variables_info) + assert result.startswith( + f"CMIP6Info(paths={expected_paths}, strict=True, alt_names=", + ) + assert result.endswith(")") def test_custom_tables_location(self): """Test constructor with custom tables location.""" @@ -95,104 +116,128 @@ def test_custom_tables_location(self): cmor_tables_path = os.path.abspath(cmor_tables_path) CMIP6Info(cmor_tables_path, default=None, strict=False) - def test_get_table_frequency(self): + def test_get_table_frequency(self, variables_info): """Test get table frequency.""" - self.assertEqual( - self.variables_info.get_table("Amon").frequency, - "mon", - ) - self.assertEqual(self.variables_info.get_table("day").frequency, "day") + assert variables_info.get_table("Amon").frequency == "mon" + assert variables_info.get_table("day").frequency == "day" - def test_get_variable_tas(self): + def test_get_variable_tas(self, variables_info): """Get tas variable.""" - var = self.variables_info.get_variable("Amon", "tas") - self.assertEqual(var.short_name, "tas") + var = variables_info.get_variable("Amon", "tas") + assert var.short_name == "tas" - def test_get_variable_from_alt_names(self): + def test_get_variable_from_alt_names(self, variables_info): """Get a variable from a known alt_names.""" - var = self.variables_info.get_variable("SImon", "sic") - self.assertEqual(var.short_name, "siconc") + var = variables_info.get_variable("SImon", "sic") + assert var.short_name == "siconc" - def test_get_variable_derived(self): + def test_get_variable_derived(self, variables_info): """Test that derived variable are looked up from other MIP tables.""" - var = self.variables_info.get_variable("3hr", "sfcWind", derived=True) - self.assertEqual(var.short_name, "sfcWind") + var = variables_info.get_variable("3hr", "sfcWind", derived=True) + assert var.short_name == "sfcWind" - def test_get_variable_from_custom(self): + def test_get_variable_from_custom(self, variables_info): """Get a variable from default.""" - self.variables_info.strict = False - var = self.variables_info.get_variable("Amon", "swcre") - self.assertEqual(var.short_name, "swcre") - self.assertEqual(var.frequency, "mon") + variables_info.strict = False + var = variables_info.get_variable("Amon", "swcre") + assert var.short_name == "swcre" + assert var.frequency == "mon" - var = self.variables_info.get_variable("day", "swcre") - self.assertEqual(var.short_name, "swcre") - self.assertEqual(var.frequency, "day") + var = variables_info.get_variable("day", "swcre") + assert var.short_name == "swcre" + assert var.frequency == "day" - def test_get_bad_variable(self): + def test_get_bad_variable(self, variables_info): """Get none if a variable is not in the given table.""" - self.assertIsNone(self.variables_info.get_variable("Omon", "ta")) + assert variables_info.get_variable("Omon", "ta") is None - def test_omon_ta_fail_if_strict(self): + def test_omon_ta_fail_if_strict(self, variables_info): """Get ta fails with Omon if strict.""" - self.assertIsNone(self.variables_info.get_variable("Omon", "ta")) + assert variables_info.get_variable("Omon", "ta") is None - def test_omon_ta_succes_if_strict(self): + def test_omon_ta_succes_if_strict(self, variables_info): """Get ta does not fail with AERMonZ if not strict.""" - self.variables_info.strict = False - var = self.variables_info.get_variable("Omon", "ta") - self.assertEqual(var.short_name, "ta") - self.assertEqual(var.frequency, "mon") + variables_info.strict = False + var = variables_info.get_variable("Omon", "ta") + assert var.short_name == "ta" + assert var.frequency == "mon" - def test_omon_toz_succes_if_strict(self): + def test_omon_toz_succes_if_strict(self, variables_info): """Get toz does not fail with Omon if not strict.""" - self.variables_info.strict = False - var = self.variables_info.get_variable("Omon", "toz") - self.assertEqual(var.short_name, "toz") - self.assertEqual(var.frequency, "mon") + variables_info.strict = False + var = variables_info.get_variable("Omon", "toz") + assert var.short_name == "toz" + assert var.frequency == "mon" - def test_get_institute_from_source(self): + def test_get_institute_from_source(self, variables_info): """Get institution for source ACCESS-CM2.""" - institute = self.variables_info.institutes["ACCESS-CM2"] - self.assertListEqual(institute, ["CSIRO-ARCCSS"]) + institute = variables_info.institutes["ACCESS-CM2"] + assert institute == ["CSIRO-ARCCSS"] - def test_get_activity_from_exp(self): + def test_get_activity_from_exp(self, variables_info): """Get activity for experiment 1pctCO2.""" - activity = self.variables_info.activities["1pctCO2"] - self.assertListEqual(activity, ["CMIP"]) + activity = variables_info.activities["1pctCO2"] + assert activity == ["CMIP"] - def test_invalid_path(self): + def test_invalid_path(self) -> None: path = Path(__file__) / "path" / "does" / "not" / "exist" msg = r"CMOR tables not found in" with pytest.raises(ValueError, match=msg): - CMIP6Info(path) + CMIP6Info(str(path)) + def test_invalid_paths(self) -> None: + path = Path(__file__) / "path" / "does" / "not" / "exist" + with pytest.raises(NotADirectoryError, match=str(path)): + CMIP6Info(paths=[path]) + + def test_no_tables_in_path(self, tmp_path: Path) -> None: + with pytest.raises(ValueError): + CMIP6Info(paths=[tmp_path]) + + def test_invalid_file(self, tmp_path: Path) -> None: + invalid_file = tmp_path / "invalid.json" + invalid_file.write_text("invalid content", encoding="utf-8") + with pytest.raises(ValueError): + CMIP6Info(paths=[tmp_path]) + + def test_invalid_file_logged( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + ) -> None: + # attach the root logger handler added by caplog + monkeypatch.setattr( + esmvalcore.cmor.table.logger, + "handlers", + logging.getLogger().handlers, + ) + invalid_file = tmp_path / "invalid.json" + invalid_file.write_text("invalid content", encoding="utf-8") + with pytest.raises(ValueError): + CMIP6Info(paths=[tmp_path]) + assert ( + f"Exception raised when loading {tmp_path}/invalid.json" + in caplog.messages + ) -class Testobs4mipsInfo(unittest.TestCase): - """Test for the obs$mips info class.""" - @classmethod - def setUpClass(cls): - """Set up tests. +class Testobs4mipsInfo: + """Tests for the obs4mips info class.""" - We read CMIP6Info once to keep tests times manageable - """ - cls.variables_info = CMIP6Info( - cmor_tables_path="obs4mips", - default=CustomInfo(), + @pytest.fixture + def variables_info(self) -> Obs4MIPsInfo: + return Obs4MIPsInfo( + paths=[ + Path("obs4mips/Tables"), + Path("cmip6-custom"), + ], strict=False, - default_table_prefix="obs4MIPs_", ) - def setUp(self): - self.variables_info.strict = False - - def test_get_table_frequency(self): + def test_get_table_frequency(self, variables_info): """Test get table frequency.""" - self.assertEqual( - self.variables_info.get_table("obs4MIPs_monStderr").frequency, - "mon", - ) + assert variables_info.get_table("monStderr").frequency == "mon" def test_custom_tables_location(self): """Test constructor with custom tables location.""" @@ -201,78 +246,77 @@ def test_custom_tables_location(self): cmor_tables_path = os.path.abspath(cmor_tables_path) CMIP6Info(cmor_tables_path, None, True) - def test_get_variable_ndvistderr(self): + def test_get_variable_ndvistderr(self, variables_info): """Get ndviStderr variable. Note table name obs4MIPs_[mip] """ - var = self.variables_info.get_variable( + var = variables_info.get_variable( "obs4MIPs_monStderr", "ndviStderr", ) - self.assertEqual(var.short_name, "ndviStderr") - self.assertEqual(var.frequency, "mon") + assert var.short_name == "ndviStderr" + assert var.frequency == "mon" - def test_get_variable_hus(self): + def test_get_variable_hus(self, variables_info): """Get hus variable.""" - var = self.variables_info.get_variable("obs4MIPs_Amon", "hus") - self.assertEqual(var.short_name, "hus") - self.assertEqual(var.frequency, "mon") + var = variables_info.get_variable("Amon", "hus") + assert var.short_name == "hus" + assert var.frequency == "mon" - def test_get_variable_hus_default_prefix(self): + def test_get_variable_hus_default_prefix(self, variables_info): """Get hus variable.""" - var = self.variables_info.get_variable("Amon", "hus") - self.assertEqual(var.short_name, "hus") - self.assertEqual(var.frequency, "mon") + var = variables_info.get_variable("Amon", "hus") + assert var.short_name == "hus" + assert var.frequency == "mon" - def test_get_variable_from_custom(self): + def test_get_variable_from_custom(self, variables_info): """Get prStderr variable. Note table name obs4MIPs_[mip] """ - var = self.variables_info.get_variable( - "obs4MIPs_monStderr", + var = variables_info.get_variable( + "monStderr", "prStderr", ) - self.assertEqual(var.short_name, "prStderr") - self.assertEqual(var.frequency, "mon") + assert var.short_name == "prStderr" + assert var.frequency == "mon" - def test_get_variable_from_custom_deriving(self): + def test_get_variable_from_custom_deriving(self, variables_info): """Get a variable from default.""" - var = self.variables_info.get_variable( - "obs4MIPs_Amon", + var = variables_info.get_variable( + "Amon", "swcre", derived=True, ) - self.assertEqual(var.short_name, "swcre") - self.assertEqual(var.frequency, "mon") + assert var.short_name == "swcre" + assert var.frequency == "mon" - var = self.variables_info.get_variable( - "obs4MIPs_Aday", + var = variables_info.get_variable( + "Aday", "swcre", derived=True, ) - self.assertEqual(var.short_name, "swcre") - self.assertEqual(var.frequency, "day") + assert var.short_name == "swcre" + assert var.frequency == "day" - def test_get_bad_variable(self): + def test_get_bad_variable(self, variables_info): """Get none if a variable is not in the given table.""" - self.assertIsNone(self.variables_info.get_variable("Omon", "tras")) - + assert variables_info.get_variable("Omon", "tras") is None -class TestCMIP5Info(unittest.TestCase): - """Test for the CMIP5 info class.""" - @classmethod - def setUpClass(cls): - """Set up tests. - - We read CMIP5Info once to keep testing times manageable - """ - cls.variables_info = CMIP5Info("cmip5", CustomInfo(), strict=True) +class TestCMIP5Info: + """Tests for the CMIP5 info class.""" - def setUp(self): - self.variables_info.strict = True + @pytest.fixture + def variables_info(self) -> CMIP5Info: + return CMIP5Info( + paths=[ + Path("cmip5/Tables"), + Path("cmip5-custom"), + ], + strict=True, + ) def test_custom_tables_location(self): """Test constructor with custom tables location.""" @@ -281,85 +325,108 @@ def test_custom_tables_location(self): cmor_tables_path = os.path.abspath(cmor_tables_path) CMIP5Info(cmor_tables_path, None, True) - def test_get_variable_tas(self): + def test_get_variable_tas(self, variables_info): """Get tas variable.""" - var = self.variables_info.get_variable("Amon", "tas") - self.assertEqual(var.short_name, "tas") + var = variables_info.get_variable("Amon", "tas") + assert var.short_name == "tas" - def test_get_variable_zg(self): + def test_get_variable_zg(self, variables_info): """Get zg variable.""" - var = self.variables_info.get_variable("Amon", "zg") - self.assertEqual(var.short_name, "zg") - self.assertEqual( - var.coordinates["plevs"].requested, - [ - "100000.", - "92500.", - "85000.", - "70000.", - "60000.", - "50000.", - "40000.", - "30000.", - "25000.", - "20000.", - "15000.", - "10000.", - "7000.", - "5000.", - "3000.", - "2000.", - "1000.", - ], - ) - - def test_get_variable_from_custom(self): + var = variables_info.get_variable("Amon", "zg") + assert var.short_name == "zg" + assert var.coordinates["plevs"].requested == [ + "100000.", + "92500.", + "85000.", + "70000.", + "60000.", + "50000.", + "40000.", + "30000.", + "25000.", + "20000.", + "15000.", + "10000.", + "7000.", + "5000.", + "3000.", + "2000.", + "1000.", + ] + + def test_get_variable_from_custom(self, variables_info): """Get a variable from default.""" - self.variables_info.strict = False - var = self.variables_info.get_variable("Amon", "swcre") - self.assertEqual(var.short_name, "swcre") - self.assertEqual(var.frequency, "mon") + variables_info.strict = False + var = variables_info.get_variable("Amon", "swcre") + assert var.short_name == "swcre" + assert var.frequency == "mon" - var = self.variables_info.get_variable("day", "swcre") - self.assertEqual(var.short_name, "swcre") - self.assertEqual(var.frequency, "day") + var = variables_info.get_variable("day", "swcre") + assert var.short_name == "swcre" + assert var.frequency == "day" - def test_get_bad_variable(self): + def test_get_bad_variable(self, variables_info): """Get none if a variable is not in the given table.""" - self.assertIsNone(self.variables_info.get_variable("Omon", "tas")) + assert variables_info.get_variable("Omon", "tas") is None - def test_aermon_ta_fail_if_strict(self): + def test_aermon_ta_fail_if_strict(self, variables_info): """Get ta fails with AERMonZ if strict.""" - self.assertIsNone(self.variables_info.get_variable("Omon", "ta")) + assert variables_info.get_variable("Omon", "ta") is None - def test_aermon_ta_succes_if_strict(self): + def test_aermon_ta_succes_if_strict(self, variables_info): """Get ta does not fail with Omon if not strict.""" - self.variables_info.strict = False - var = self.variables_info.get_variable("Omon", "ta") - self.assertEqual(var.short_name, "ta") - self.assertEqual(var.frequency, "mon") + variables_info.strict = False + var = variables_info.get_variable("Omon", "ta") + assert var.short_name == "ta" + assert var.frequency == "mon" - def test_omon_toz_succes_if_strict(self): + def test_omon_toz_succes_if_strict(self, variables_info): """Get toz does not fail with Omon if not strict.""" - self.variables_info.strict = False - var = self.variables_info.get_variable("Omon", "toz") - self.assertEqual(var.short_name, "toz") - self.assertEqual(var.frequency, "mon") - - -class TestCMIP3Info(unittest.TestCase): - """Test for the CMIP5 info class.""" + variables_info.strict = False + var = variables_info.get_variable("Omon", "toz") + assert var.short_name == "toz" + assert var.frequency == "mon" + + def test_invalid_file(self, tmp_path: Path) -> None: + invalid_file = tmp_path / "invalid" + invalid_file.write_text("invalid content", encoding="utf-8") + with pytest.raises(ValueError): + CMIP5Info(paths=[tmp_path]) + + def test_invalid_file_logged( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + ) -> None: + # attach the root logger handler added by caplog + monkeypatch.setattr( + esmvalcore.cmor.table.logger, + "handlers", + logging.getLogger().handlers, + ) + invalid_file = tmp_path / "invalid" + invalid_file.write_text("invalid content", encoding="utf-8") + with pytest.raises(ValueError): + CMIP5Info(paths=[tmp_path]) + assert ( + f"Exception raised when loading {tmp_path}/invalid" + in caplog.messages + ) - @classmethod - def setUpClass(cls): - """Set up tests. - We read CMIP5Info once to keep testing times manageable - """ - cls.variables_info = CMIP3Info("cmip3", CustomInfo(), strict=True) +class TestCMIP3Info: + """Tests for the CMIP3 info class.""" - def setUp(self): - self.variables_info.strict = True + @pytest.fixture + def variables_info(self) -> CMIP3Info: + return CMIP3Info( + paths=[ + Path("cmip3/Tables"), + Path("cmip3-custom"), + ], + strict=True, + ) def test_custom_tables_location(self): """Test constructor with custom tables location.""" @@ -368,82 +435,81 @@ def test_custom_tables_location(self): cmor_tables_path = os.path.abspath(cmor_tables_path) CMIP3Info(cmor_tables_path, None, True) - def test_get_variable_tas(self): + def test_get_variable_tas(self, variables_info): """Get tas variable.""" - var = self.variables_info.get_variable("A1", "tas") - self.assertEqual(var.short_name, "tas") + var = variables_info.get_variable("A1", "tas") + assert var.short_name == "tas" - def test_get_variable_zg(self): + def test_get_variable_zg(self, variables_info): """Get zg variable.""" - var = self.variables_info.get_variable("A1", "zg") - self.assertEqual(var.short_name, "zg") - self.assertEqual( - var.coordinates["pressure"].requested, - [ - "100000.", - "92500.", - "85000.", - "70000.", - "60000.", - "50000.", - "40000.", - "30000.", - "25000.", - "20000.", - "15000.", - "10000.", - "7000.", - "5000.", - "3000.", - "2000.", - "1000.", - ], - ) - - def test_get_variable_from_custom(self): + var = variables_info.get_variable("A1", "zg") + assert var.short_name == "zg" + assert var.coordinates["pressure"].requested == [ + "100000.", + "92500.", + "85000.", + "70000.", + "60000.", + "50000.", + "40000.", + "30000.", + "25000.", + "20000.", + "15000.", + "10000.", + "7000.", + "5000.", + "3000.", + "2000.", + "1000.", + ] + + def test_get_variable_from_custom(self, variables_info): """Get a variable from default.""" - self.variables_info.strict = False - var = self.variables_info.get_variable("A1", "swcre") - self.assertEqual(var.short_name, "swcre") - self.assertEqual(var.frequency, "") + variables_info.strict = False + var = variables_info.get_variable("A1", "swcre") + assert var.short_name == "swcre" + assert var.frequency == "" - var = self.variables_info.get_variable("day", "swcre") - self.assertEqual(var.short_name, "swcre") - self.assertEqual(var.frequency, "") + var = variables_info.get_variable("day", "swcre") + assert var.short_name == "swcre" + assert var.frequency == "" - def test_get_bad_variable(self): + def test_get_bad_variable(self, variables_info): """Get none if a variable is not in the given table.""" - self.assertIsNone(self.variables_info.get_variable("O1", "tas")) + assert variables_info.get_variable("O1", "tas") is None - def test_aermon_ta_fail_if_strict(self): + def test_aermon_ta_fail_if_strict(self, variables_info): """Get ta fails with AERMonZ if strict.""" - self.assertIsNone(self.variables_info.get_variable("O1", "ta")) + assert variables_info.get_variable("O1", "ta") is None - def test_aermon_ta_succes_if_strict(self): + def test_aermon_ta_succes_if_strict(self, variables_info): """Get ta does not fail with Omon if not strict.""" - self.variables_info.strict = False - var = self.variables_info.get_variable("O1", "ta") - self.assertEqual(var.short_name, "ta") - self.assertEqual(var.frequency, "") - - def test_omon_toz_succes_if_strict(self): - """Get toz does not fail with Omon if not strict.""" - self.variables_info.strict = False - var = self.variables_info.get_variable("O1", "toz") - self.assertEqual(var.short_name, "toz") - self.assertEqual(var.frequency, "") - - -class TestCORDEXInfo(unittest.TestCase): - """Test for the CORDEX info class.""" - - @classmethod - def setUpClass(cls): - """Set up tests. - - We read CORDEX once to keep testing times manageable - """ - cls.variables_info = CMIP5Info("cordex", default=CustomInfo()) + variables_info.strict = False + var = variables_info.get_variable("O1", "ta") + assert var.short_name == "ta" + assert var.frequency == "" + + def test_derived_variable_success_if_strict(self, variables_info): + """Get swcre does not fail with Omon if not strict.""" + variables_info.strict = False + var = variables_info.get_variable("O1", "swcre") + assert var.short_name == "swcre" + assert var.frequency == "" + + +class TestCORDEXInfo: + """Tests for the CORDEX info class.""" + + @pytest.fixture + def variables_info(self) -> CMIP5Info: + return CMIP5Info( + paths=[ + Path("cordex/Tables"), + Path("cordex-custom"), + Path("cmip5-custom"), + ], + ) def test_custom_tables_location(self): """Test constructor with custom tables location.""" @@ -451,116 +517,139 @@ def test_custom_tables_location(self): cmor_tables_path = os.path.join(cmor_path, "tables", "cordex") CMIP5Info(cmor_tables_path) - def test_get_variable_tas(self): + def test_get_variable_tas(self, variables_info): """Get tas variable.""" - var = self.variables_info.get_variable("mon", "tas") - self.assertEqual(var.short_name, "tas") + var = variables_info.get_variable("mon", "tas") + assert var.short_name == "tas" - def test_get_bad_variable(self): + def test_get_bad_variable(self, variables_info): """Get none if a variable is not in the given table.""" - self.assertIsNone(self.variables_info.get_variable("Omon", "tas")) + assert variables_info.get_variable("Omon", "tas") is None -class TestCustomInfo(unittest.TestCase): - """Test for the custom info class.""" +class TestCustomInfo: + """Tests for the custom info class.""" - @classmethod - def setUpClass(cls): - """Set up tests. + @pytest.fixture + def variables_info(self) -> CustomInfo: + return CustomInfo() - We read CMIP5Info once to keep testing times manageable - """ - cls.variables_info = CustomInfo() + def test_repr(self, variables_info: CustomInfo) -> None: + builtin_tables_path = Path(esmvalcore.cmor.__file__).parent / "tables" + expected_paths = [ + builtin_tables_path / "old-custom-coordinates", + builtin_tables_path / "cmip5-custom", + ] + result = repr(variables_info) + assert result == f"CustomInfo(paths={expected_paths})" - def test_custom_tables_default_location(self): + def test_custom_tables_default_location(self, variables_info): """Test constructor with default tables location.""" custom_info = CustomInfo() - expected_cmor_folder = os.path.join( - os.path.dirname(esmvalcore.cmor.__file__), - "tables", - "custom", + builtin_tables_path = Path(esmvalcore.cmor.__file__).parent / "tables" + default_paths = ( + builtin_tables_path / "old-custom-coordinates", + builtin_tables_path / "cmip5-custom", ) - self.assertEqual(custom_info._cmor_folder, expected_cmor_folder) - self.assertTrue(custom_info.tables["custom"]) - self.assertTrue(custom_info.coords) + assert custom_info.paths == default_paths + assert custom_info.tables["custom"] + assert custom_info.coords - def test_custom_tables_location(self): + def test_custom_tables_location(self, variables_info): """Test constructor with custom tables location.""" cmor_path = os.path.dirname(os.path.realpath(esmvalcore.cmor.__file__)) - default_cmor_tables_path = os.path.join(cmor_path, "tables", "custom") cmor_tables_path = os.path.join(cmor_path, "tables", "cmip5") cmor_tables_path = os.path.abspath(cmor_tables_path) custom_info = CustomInfo(cmor_tables_path) - self.assertEqual(custom_info._cmor_folder, default_cmor_tables_path) - self.assertEqual(custom_info._user_table_folder, cmor_tables_path) - self.assertTrue(custom_info.tables["custom"]) - self.assertTrue(custom_info.coords) + builtin_tables_path = Path(esmvalcore.cmor.__file__).parent / "tables" + default_paths = ( + builtin_tables_path / "old-custom-coordinates", + builtin_tables_path / "cmip5-custom", + ) + assert custom_info.paths == (*default_paths, Path(cmor_tables_path)) + assert custom_info.tables["custom"] + assert custom_info.coords def test_custom_tables_invalid_location(self): """Test constructor with invalid custom tables location.""" - with self.assertRaises(ValueError): + with pytest.raises(ValueError): CustomInfo("this_file_does_not_exist.dat") - def test_get_variable_netcre(self): + def test_get_variable_netcre(self, variables_info): """Get tas variable.""" CustomInfo() - var = self.variables_info.get_variable("Amon", "netcre") - self.assertEqual(var.short_name, "netcre") + var = variables_info.get_variable("Amon", "netcre") + assert var.short_name == "netcre" - def test_get_bad_variable(self): + def test_get_bad_variable(self, variables_info): """Get none if a variable is not in the given table.""" - self.assertIsNone(self.variables_info.get_variable("Omon", "badvar")) + assert variables_info.get_variable("Omon", "badvar") is None - def test_get_variable_tasconf5(self): + def test_get_variable_tasconf5(self, variables_info): """Get tas variable.""" CustomInfo() - var = self.variables_info.get_variable("Amon", "tasConf5") - self.assertEqual(var.short_name, "tasConf5") - self.assertEqual( - var.long_name, - "Near-Surface Air Temperature Uncertainty Range", + var = variables_info.get_variable("Amon", "tasConf5") + assert var.short_name == "tasConf5" + assert ( + var.long_name == "Near-Surface Air Temperature Uncertainty Range" ) - self.assertEqual(var.units, "K") + assert var.units == "K" - def test_get_variable_tasconf95(self): + def test_get_variable_tasconf95(self, variables_info): """Get tas variable.""" CustomInfo() - var = self.variables_info.get_variable("Amon", "tasConf95") - self.assertEqual(var.short_name, "tasConf95") - self.assertEqual( - var.long_name, - "Near-Surface Air Temperature Uncertainty Range", + var = variables_info.get_variable("Amon", "tasConf95") + assert var.short_name == "tasConf95" + assert ( + var.long_name == "Near-Surface Air Temperature Uncertainty Range" ) - self.assertEqual(var.units, "K") + assert var.units == "K" - def test_get_variable_tasaga(self): + def test_get_variable_tasaga(self, variables_info): """Get tas variable.""" CustomInfo() - var = self.variables_info.get_variable("Amon", "tasaga") - self.assertEqual(var.short_name, "tasaga") - self.assertEqual( - var.long_name, - "Global-mean Near-Surface Air Temperature Anomaly", + var = variables_info.get_variable("Amon", "tasaga") + assert var.short_name == "tasaga" + assert ( + var.long_name == "Global-mean Near-Surface Air Temperature Anomaly" ) - self.assertEqual(var.units, "K") + assert var.units == "K" - def test_get_variable_ch4s(self): + def test_get_variable_ch4s(self, variables_info): """Get ch4s variable.""" CustomInfo() - var = self.variables_info.get_variable("Amon", "ch4s") - self.assertEqual(var.short_name, "ch4s") - self.assertEqual(var.long_name, "Atmosphere CH4 surface") - self.assertEqual(var.units, "1e-09") + var = variables_info.get_variable("Amon", "ch4s") + assert var.short_name == "ch4s" + assert var.long_name == "Atmosphere CH4 surface" + assert var.units == "1e-09" - def test_get_variable_tosstderr(self): + def test_get_variable_tosstderr(self, variables_info): """Get tosStderr variable.""" CustomInfo() - var = self.variables_info.get_variable("Omon", "tosStderr") - self.assertEqual(var.short_name, "tosStderr") - self.assertEqual(var.long_name, "Sea Surface Temperature Error") - self.assertEqual(var.units, "K") + var = variables_info.get_variable("Omon", "tosStderr") + assert var.short_name == "tosStderr" + assert var.long_name == "Sea Surface Temperature Error" + assert var.units == "K" + + +class TestNoInfo: + """Tests for the no info class.""" + + @pytest.fixture + def variables_info(self) -> NoInfo: + return NoInfo() + + def test_repr(self, variables_info: NoInfo) -> None: + result = repr(variables_info) + assert result == "NoInfo()" + + def test_get_variable_tas(self, variables_info: NoInfo) -> None: + """Get tas variable.""" + var = variables_info.get_variable("Amon", "tas") + assert isinstance(var, VariableInfo) + assert var.short_name == "tas" @pytest.mark.parametrize( diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 3fae8cd8e6..ff8ec4fdc7 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,22 +1,20 @@ from __future__ import annotations import os -from pathlib import Path from typing import TYPE_CHECKING import iris import pytest -import esmvalcore.io.local from esmvalcore.io.local import ( + LocalDataSource, LocalFile, - _replace_tags, _select_files, ) -from esmvalcore.local import _select_drs if TYPE_CHECKING: from collections.abc import Callable, Iterator + from pathlib import Path from esmvalcore.typing import Facets, FacetValue @@ -36,11 +34,12 @@ def create_test_file(filename, tracking_id=None): def _get_files( # noqa: C901,PLR0912 + self: LocalDataSource, root_path: Path, facets: Facets, tracking_id: Iterator[int], suffix: str = "nc", -) -> tuple[list[LocalFile], list[Path]]: +) -> list[LocalFile]: """Return dummy files. Wildcards are only supported for `dataset` and `institute`; in this case @@ -55,34 +54,15 @@ def _get_files( # noqa: C901,PLR0912 else: all_facets = [facets] - # Globs without expanded facets - dir_template = _select_drs("input_dir", facets["project"], "default") # type: ignore[arg-type] - file_template = _select_drs("input_file", facets["project"], "default") # type: ignore[arg-type] - dir_globs = _replace_tags(dir_template, facets) - file_globs = _replace_tags(file_template, facets) - globs = sorted( - root_path / "input" / d / f for d in dir_globs for f in file_globs - ) + self.rootpath = root_path / "input" + facets = dict(facets) + if "original_short_name" in facets: + facets["short_name"] = facets["original_short_name"] files = [] for expanded_facets in all_facets: filenames = [] - dir_template = _select_drs( - "input_dir", - expanded_facets["project"], # type: ignore[arg-type] - "default", - ) - file_template = _select_drs( - "input_file", - expanded_facets["project"], # type: ignore[arg-type] - "default", - ) - - dir_globs = _replace_tags(dir_template, expanded_facets) - file_globs = _replace_tags(file_template, expanded_facets) - filename = str( - root_path / "input" / dir_globs[0] / Path(file_globs[0]).name, - ) + filename = str(self._get_glob_patterns(**expanded_facets)[0]) if filename.endswith("nc"): filename = f"{filename[:-2]}{suffix}" @@ -120,7 +100,7 @@ def _get_files( # noqa: C901,PLR0912 if "timerange" in facets: files = _select_files(files, facets["timerange"]) - return files, globs + return files def _tracking_ids(i=0): @@ -129,37 +109,28 @@ def _tracking_ids(i=0): i += 1 -def _get_find_files_func( +def _get_find_data_func( path: Path, suffix: str = "nc", -) -> Callable[ - ..., - tuple[list[LocalFile], list[Path]] | list[LocalFile], -]: +) -> Callable[..., list[LocalFile]]: tracking_id = _tracking_ids() - def find_files( - self: esmvalcore.io.local.LocalDataSource, - *, - debug: bool = False, + def find_data( + self: LocalDataSource, **facets: FacetValue, - ) -> tuple[list[LocalFile], list[Path]] | list[LocalFile]: - files, file_globs = _get_files(path, facets, tracking_id, suffix) - if debug: - return files, file_globs - return files + ) -> list[LocalFile]: + return _get_files(self, path, facets, tracking_id, suffix) - return find_files + return find_data @pytest.fixture -def patched_datafinder(tmp_path: Path, monkeypatch: pytest.MonkeyPath) -> None: - find_files = _get_find_files_func(tmp_path) - monkeypatch.setattr( - esmvalcore.io.local.LocalDataSource, - "find_data", - find_files, - ) +def patched_datafinder( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + find_data = _get_find_data_func(tmp_path) + monkeypatch.setattr(LocalDataSource, "find_data", find_data) @pytest.fixture @@ -167,12 +138,8 @@ def patched_datafinder_grib( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - find_files = _get_find_files_func(tmp_path, suffix="grib") - monkeypatch.setattr( - esmvalcore.io.local.LocalDataSource, - "find_data", - find_files, - ) + find_data = _get_find_data_func(tmp_path, suffix="grib") + monkeypatch.setattr(LocalDataSource, "find_data", find_data) @pytest.fixture @@ -191,25 +158,17 @@ def patched_failing_datafinder( """ tracking_id = _tracking_ids() - def find_files( - self: esmvalcore.io.local.LocalDataSource, - *, - debug: bool = False, + def find_data( + self: LocalDataSource, **facets: FacetValue, - ) -> tuple[list[LocalFile], list[Path]] | list[LocalFile]: - files, file_globs = _get_files(tmp_path, facets, tracking_id) + ) -> list[LocalFile]: + files = _get_files(self, tmp_path, facets, tracking_id) if facets["frequency"] == "fx": files = [] returned_files = [] for file in files: if not ("AAA" in file.name and "rsutcs" in file.name): returned_files.append(file) - if debug: - return returned_files, file_globs return returned_files - monkeypatch.setattr( - esmvalcore.io.local.LocalDataSource, - "find_data", - find_files, - ) + monkeypatch.setattr(LocalDataSource, "find_data", find_data) diff --git a/tests/integration/io/test_local.py b/tests/integration/io/test_local.py index 81a536f196..71b114f35f 100644 --- a/tests/integration/io/test_local.py +++ b/tests/integration/io/test_local.py @@ -9,6 +9,10 @@ import pytest import yaml +import esmvalcore +import esmvalcore.cmor.table +import esmvalcore.config._config +import esmvalcore.local from esmvalcore.config import CFG from esmvalcore.io.local import ( LocalDataSource, @@ -66,13 +70,35 @@ def create_tree(path, filenames=None, symlinks=None): @pytest.mark.parametrize("cfg", CONFIG["get_output_file"]) -def test_get_output_file(cfg): +def test_get_output_file(monkeypatch, cfg): """Test getting output name for preprocessed files.""" + monkeypatch.setattr(esmvalcore.cmor.table, "CMOR_TABLES", {}) + monkeypatch.setitem( + CFG, + "config_developer_file", + Path(esmvalcore.__path__[0], "config-developer.yml"), + ) output_file = _get_output_file(cfg["variable"], cfg["preproc_dir"]) expected = Path(cfg["output_file"]) assert output_file == expected +@pytest.mark.parametrize("cfg", CONFIG["get_output_file"]) +def test_get_output_file_no_config_developer(monkeypatch, cfg): + """Test getting output name for preprocessed files.""" + monkeypatch.setattr(esmvalcore.local, "CFG", {}) + monkeypatch.setattr(esmvalcore.cmor.table, "CMOR_TABLES", {}) + monkeypatch.setattr(esmvalcore.config._config, "CFG", {}) + monkeypatch.setattr(esmvalcore.local, "CONFIG_DEVELOPER", {}) + output_file = _get_output_file(cfg["variable"], cfg["preproc_dir"]) + expected = Path(cfg["output_file"]) + assert output_file == expected + # This test ensures that only the directory structure bits of + # config-developer.yml have been loaded. + assert esmvalcore.config._config.CFG + assert not esmvalcore.cmor.table.CMOR_TABLES + + @pytest.fixture def root(tmp_path): """Root function for tests.""" @@ -83,7 +109,7 @@ def root(tmp_path): @pytest.mark.parametrize("cfg", CONFIG["get_input_filelist"]) -def test_find_files(monkeypatch, root, cfg): +def test_find_files(monkeypatch, root, cfg, mocker): """Test retrieving input filelist.""" if "drs" not in cfg: pytest.skip( @@ -95,6 +121,13 @@ def test_find_files(monkeypatch, root, cfg): pprint.pformat(cfg["variable"]), ) project = cfg["variable"]["project"] + mocker.patch.object(esmvalcore.local, "_ensure_config_developer_drs") + monkeypatch.setattr(esmvalcore.cmor.table, "CMOR_TABLES", {}) + monkeypatch.setitem( + CFG, + "config_developer_file", + Path(esmvalcore.__path__[0], "config-developer.yml"), + ) monkeypatch.setitem(CFG, "drs", {project: cfg["drs"]}) monkeypatch.setitem(CFG, "rootpath", {project: root}) create_tree( @@ -112,6 +145,7 @@ def test_find_files(monkeypatch, root, cfg): ] assert [Path(f) for f in input_filelist] == sorted(ref_files) assert [Path(g) for g in globs] == sorted(ref_globs) + esmvalcore.local._ensure_config_developer_drs.assert_called_once() def test_find_files_with_facets(monkeypatch, root): @@ -121,6 +155,12 @@ def test_find_files_with_facets(monkeypatch, root): break project = cfg["variable"]["project"] + monkeypatch.setattr(esmvalcore.cmor.table, "CMOR_TABLES", {}) + monkeypatch.setitem( + CFG, + "config_developer_file", + Path(esmvalcore.__path__[0], "config-developer.yml"), + ) monkeypatch.setitem(CFG, "drs", {project: cfg["drs"]}) monkeypatch.setitem(CFG, "rootpath", {project: root}) @@ -176,7 +216,13 @@ def test_find_data(root, cfg): assert str(pattern) in data_source.debug_info -def test_select_invalid_drs_structure(): +def test_select_invalid_drs_structure(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(esmvalcore.cmor.table, "CMOR_TABLES", {}) + monkeypatch.setitem( + CFG, + "config_developer_file", + Path(esmvalcore.__path__[0], "config-developer.yml"), + ) msg = ( r"drs _INVALID_STRUCTURE_ for CMIP6 project not specified in " r"config-developer file" diff --git a/tests/integration/recipe/test_recipe.py b/tests/integration/recipe/test_recipe.py index b87a696387..7799ba522f 100644 --- a/tests/integration/recipe/test_recipe.py +++ b/tests/integration/recipe/test_recipe.py @@ -3294,7 +3294,7 @@ def test_bias_two_refs_with_mmm(tmp_path, patched_datafinder, session): additional_datasets: - {dataset: CanESM5, group: ref, reference_for_bias: true} - {dataset: CESM2, group: ref, reference_for_bias: true} - - {dataset: MPI-ESM-LR, group: notref} + - {dataset: MPI-ESM1-2-LR, group: notref} scripts: null """) @@ -3562,7 +3562,7 @@ def test_distance_metrics_two_refs_with_mmm( additional_datasets: - {dataset: CESM2, ensemble: r1i1p1f1, reference_for_metric: true} - {dataset: CESM2, ensemble: r2i1p1f1, reference_for_metric: true} - - {dataset: MPI-ESM-LR} + - {dataset: MPI-ESM1-2-LR} scripts: null """) @@ -3825,9 +3825,9 @@ def test_align_metadata_invalid_project(tmp_path, patched_datafinder, session): """) msg = ( "align_metadata failed: \"No CMOR tables available for project 'ZZZ'. " - "The following tables are available: custom, CMIP7, CMIP6, CMIP5, " - "CMIP3, OBS, OBS6, native6, obs4MIPs, ana4MIPs, EMAC, CORDEX, IPSLCM, " - 'ICON, CESM, ACCESS."' + "The following tables are available: CMIP7, CMIP6, CMIP5, CMIP3, " + "CORDEX, obs4MIPs, ana4MIPs, native6, ACCESS, CESM, EMAC, ICON, IPSLCM, " + 'OBS6, OBS."' ) with pytest.raises(RecipeError) as exc: get_recipe(tmp_path, content, session) diff --git a/tests/unit/cmor/test_table.py b/tests/unit/cmor/test_table.py index 155933d958..81a286a95a 100644 --- a/tests/unit/cmor/test_table.py +++ b/tests/unit/cmor/test_table.py @@ -18,6 +18,11 @@ def setUp(self): "dim2": CoordinateInfo("dim2"), } + def test_repr(self) -> None: + """Test __repr__.""" + result = repr(self.info) + assert result == "" + def test_constructor(self): """Test basic constructor.""" self.assertEqual("table_type", self.info.table_type) @@ -117,8 +122,10 @@ def test_read_standard_name(self): def test_read_var_name(self): """Test var_name.""" + # There does not appear to be a "var_name" field in any of the CMOR + # tables. Could the var_name attribute be removed? info = CoordinateInfo("var") - info.read_json({"var_name": self.value}) + info.read_json({"out_name": self.value}) self.assertEqual(info.var_name, self.value) def test_read_out_name(self): diff --git a/tests/unit/config/test_config.py b/tests/unit/config/test_config.py index 035ccbd7f2..20ec067ab4 100644 --- a/tests/unit/config/test_config.py +++ b/tests/unit/config/test_config.py @@ -6,6 +6,7 @@ import pytest import yaml +import esmvalcore.cmor.table import esmvalcore.config._config from esmvalcore.cmor.check import CheckLevels from esmvalcore.config import CFG, _config, _config_validators @@ -161,14 +162,10 @@ def test_get_project_config(mocker): def test_load_default_config(cfg_default, monkeypatch): """Test that the default configuration can be loaded.""" - project_cfg = {} - monkeypatch.setattr(_config, "CFG", project_cfg) root_path = importlib_files("esmvalcore") - default_dev_file = root_path / "config-developer.yml" - config_dir = root_path / "config" / "configurations" / "defaults" + default_config_dir = root_path / "config" / "configurations" / "defaults" default_project_settings = dask.config.collect( - paths=[str(p) for p in config_dir.glob("extra_facets_*.yml")] - + [str(config_dir / "preprocessor_filename_template.yml")], + paths=[str(p) for p in default_config_dir.glob("*.yml")], env={}, )["projects"] @@ -178,7 +175,6 @@ def test_load_default_config(cfg_default, monkeypatch): "auxiliary_data_dir": Path.home() / "auxiliary_data", "check_level": CheckLevels.DEFAULT, "compress_netcdf": False, - "config_developer_file": default_dev_file, "dask": { "profiles": { "local_threaded": { @@ -261,9 +257,6 @@ def test_load_default_config(cfg_default, monkeypatch): assert getattr(session, path + "_dir") == session.session_dir / path assert session.plot_dir == session.session_dir / "plots" - # Check that projects were configured - assert project_cfg - def test_rootpath_obs4mips_case_correction(monkeypatch): """Test that the name of the obs4MIPs project is correct in rootpath.""" @@ -328,8 +321,14 @@ def test_get_ignored_warnings_none(project, step): assert get_ignored_warnings(project, step) is None -def test_get_ignored_warnings_emac(): +def test_get_ignored_warnings_emac(monkeypatch: pytest.MonkeyPatch) -> None: """Test ``get_ignored_warnings``.""" + monkeypatch.setattr(esmvalcore.cmor.table, "CMOR_TABLES", {}) + monkeypatch.setitem( + CFG, + "config_developer_file", + Path(esmvalcore.__path__[0], "config-developer.yml"), + ) ignored_warnings = get_ignored_warnings("EMAC", "load") assert isinstance(ignored_warnings, list) assert ignored_warnings diff --git a/tests/unit/config/test_config_object.py b/tests/unit/config/test_config_object.py index 83f9b4e64e..3ecbe5db68 100644 --- a/tests/unit/config/test_config_object.py +++ b/tests/unit/config/test_config_object.py @@ -5,6 +5,7 @@ import pytest import esmvalcore +import esmvalcore.cmor.table import esmvalcore.config._config_object from esmvalcore.config import CFG, Config, Session from esmvalcore.config._config_object import DEFAULT_CONFIG_DIR @@ -44,6 +45,48 @@ def test_config_update(): config.update(fail_dict) +@pytest.mark.parametrize("update_format", ["mapping", "kwargs", "tuple"]) +def test_config_update_config_developer_set_last( + monkeypatch: pytest.MonkeyPatch, + update_format: str, +) -> None: + monkeypatch.setattr(esmvalcore.cmor.table, "CMOR_TABLES", {}) + new_config = { + "config_developer_file": Path(esmvalcore.__file__).parent + / "config-developer.yml", + "projects": { + "CMIP6": { + "cmor_table": { + "type": "esmvalcore.cmor.table.NoInfo", + }, + }, + }, + } + config = Config({"output_dir": "directory"}) + if update_format == "mapping": + config.update(new_config) + elif update_format == "kwargs": + config.update(**new_config) + elif update_format == "tuple": + config.update(tuple(new_config.items())) + + assert len(esmvalcore.cmor.table.CMOR_TABLES) > 1 + assert "CMIP6" in esmvalcore.cmor.table.CMOR_TABLES + assert isinstance( + esmvalcore.cmor.table.CMOR_TABLES["CMIP6"], + esmvalcore.cmor.table.CMIP6Info, + ) + + +def test_config_update_too_many_args() -> None: + config = Config({"output_dir": "directory"}) + with pytest.raises( + TypeError, + match=r"Expected at most 1 positional argument, got 2", + ): + config.update(1, 2) + + def test_set_bad_item(): config = Config({"output_dir": "config"}) with pytest.raises(InvalidConfigParameter) as err_exc: diff --git a/tests/unit/config/test_config_validator.py b/tests/unit/config/test_config_validator.py index eed9b19bd5..8d3e594f02 100644 --- a/tests/unit/config/test_config_validator.py +++ b/tests/unit/config/test_config_validator.py @@ -5,6 +5,7 @@ import yaml import esmvalcore +import esmvalcore.cmor.table from esmvalcore import __version__ as current_version from esmvalcore.config import CFG from esmvalcore.config._config_validators import ( @@ -228,10 +229,26 @@ def generate_validator_testcases(valid): { "validator": validate_projects, "success": ( - ({"CMIP6": {}}, {"CMIP6": {}}), + ( + {"CMIP6": {}}, + { + "CMIP6": { + "cmor_table": { + "type": "esmvalcore.cmor.table.NoInfo", + }, + }, + }, + ), ( {"CMIP6": {"extra_facets": {}}}, - {"CMIP6": {"extra_facets": {}}}, + { + "CMIP6": { + "cmor_table": { + "type": "esmvalcore.cmor.table.NoInfo", + }, + "extra_facets": {}, + }, + }, ), ), "fail": ( @@ -311,13 +328,14 @@ def test_handle_deprecation(remove_version): def test_validate_config_developer_none(): """Test ``validate_config_developer``.""" path = validate_config_developer(None) - assert path == Path(esmvalcore.__file__).parent / "config-developer.yml" + assert path is None -def test_validate_config_developer(tmp_path): +def test_validate_config_developer(tmp_path, monkeypatch): """Test ``validate_config_developer``.""" + monkeypatch.setattr(esmvalcore.cmor.table, "CMOR_TABLES", {}) custom_table_path = ( - Path(esmvalcore.__file__).parent / "cmor" / "tables" / "custom" + Path(esmvalcore.__file__).parent / "cmor" / "tables" / "cmip5-custom" ) cfg_dev = { "custom": {"cmor_path": custom_table_path}, diff --git a/tests/unit/config/test_data_sources.py b/tests/unit/config/test_data_sources.py index b1e08f69af..1f80a69614 100644 --- a/tests/unit/config/test_data_sources.py +++ b/tests/unit/config/test_data_sources.py @@ -1,9 +1,11 @@ from __future__ import annotations +from pathlib import Path from typing import TYPE_CHECKING import pytest +import esmvalcore.cmor.table import esmvalcore.config._data_sources import esmvalcore.local from esmvalcore.exceptions import InvalidConfigParameter @@ -37,6 +39,12 @@ def test_load_legacy_data_sources( session["projects"][project].pop("data", None) session["search_esgf"] = search_esgf session["download_dir"] = "~/climate_data" + monkeypatch.setattr(esmvalcore.cmor.table, "CMOR_TABLES", {}) + monkeypatch.setitem( + esmvalcore.local.CFG, + "config_developer_file", + Path(esmvalcore.__path__[0], "config-developer.yml"), + ) monkeypatch.setitem( esmvalcore.local.CFG, "rootpath", diff --git a/tests/unit/io/local/test_get_data_sources.py b/tests/unit/io/local/test_get_data_sources.py index 8f19709b8a..d3ccf59e71 100644 --- a/tests/unit/io/local/test_get_data_sources.py +++ b/tests/unit/io/local/test_get_data_sources.py @@ -5,8 +5,9 @@ import pytest +import esmvalcore +import esmvalcore.cmor.table from esmvalcore.config import CFG -from esmvalcore.config._config_validators import validate_config_developer from esmvalcore.io.local import LocalDataSource from esmvalcore.local import DataSource, _get_data_sources @@ -33,7 +34,12 @@ ) def test_get_data_sources(monkeypatch, rootpath_drs): # Make sure that default config-developer file is used - validate_config_developer(None) + monkeypatch.setattr(esmvalcore.cmor.table, "CMOR_TABLES", {}) + monkeypatch.setitem( + CFG, + "config_developer_file", + Path(esmvalcore.__path__[0], "config-developer.yml"), + ) rootpath, drs = rootpath_drs monkeypatch.setitem(CFG, "rootpath", rootpath) @@ -48,7 +54,12 @@ def test_get_data_sources(monkeypatch, rootpath_drs): def test_get_data_sources_nodefault(monkeypatch): # Make sure that default config-developer file is used - validate_config_developer(None) + monkeypatch.setattr(esmvalcore.cmor.table, "CMOR_TABLES", {}) + monkeypatch.setitem( + CFG, + "config_developer_file", + Path(esmvalcore.__path__[0], "config-developer.yml"), + ) monkeypatch.setitem( CFG, diff --git a/tests/unit/preprocessor/test_configuration.py b/tests/unit/preprocessor/test_configuration.py index 454b9514a2..b89ae9ad73 100644 --- a/tests/unit/preprocessor/test_configuration.py +++ b/tests/unit/preprocessor/test_configuration.py @@ -2,10 +2,14 @@ from __future__ import annotations +from pathlib import Path from typing import TYPE_CHECKING import pytest +import esmvalcore +import esmvalcore.cmor.table +from esmvalcore.config import CFG from esmvalcore.dataset import Dataset from esmvalcore.exceptions import RecipeError from esmvalcore.preprocessor import ( @@ -147,9 +151,16 @@ def test_get_preprocessor_filename_default( def test_get_preprocessor_filename_falls_back_to_config_developer( + monkeypatch: pytest.MonkeyPatch, session: Session, ) -> None: """Test the function `_get_preprocessor_filename`.""" + monkeypatch.setattr(esmvalcore.cmor.table, "CMOR_TABLES", {}) + monkeypatch.setitem( + CFG, + "config_developer_file", + Path(esmvalcore.__path__[0], "config-developer.yml"), + ) session["projects"]["CMIP6"].pop("preprocessor_filename_template") dataset = Dataset( project="CMIP6",