Implement implicit DD version conversion in NetCDF backend

Author: maarten-ic

imas/backends/netcdf/db_entry_nc.py

@@ -11,7 +11,7 @@
 from imas.backends.netcdf.ids2nc import IDS2NC
 from imas.backends.netcdf.nc2ids import NC2IDS
 from imas.exception import DataEntryException, InvalidNetCDFEntry
-from imas.ids_convert import NBCPathMap, convert_ids
+from imas.ids_convert import NBCPathMap, dd_version_map_from_factories
 from imas.ids_factory import IDSFactory
 from imas.ids_toplevel import IDSToplevel
 
@@ -123,14 +123,19 @@ def get(
 
         # Load data into the destination IDS
         if self._ds_factory.dd_version == destination._dd_version:
-            NC2IDS(group, destination).run()
+            NC2IDS(group, destination, destination.metadata, None).run()
         else:
-            # FIXME: implement automatic conversion using nbc_map
-            #   As a work-around: do an explicit conversion, but automatic conversion
-            #   will also be needed to implement lazy loading.
-            ids = self._ds_factory.new(ids_name)
-            NC2IDS(group, ids).run()
-            convert_ids(ids, None, target=destination)
+            # Construct relevant NBCPathMap, the one we get from DBEntry has the reverse
+            # mapping from what we need. The imas_core logic does the mapping from
+            # in-memory to on-disk, while we take what is on-disk and map it to
+            # in-memory.
+            ddmap, source_is_older = dd_version_map_from_factories(
+                ids_name, self._ds_factory, self._factory
+            )
+            nbc_map = ddmap.old_to_new if source_is_older else ddmap.new_to_old
+            NC2IDS(
+                group, destination, self._ds_factory.new(ids_name).metadata, nbc_map
+            ).run()
 
         return destination
 

imas/backends/netcdf/nc2ids.py

@@ -8,6 +8,7 @@
 from imas.backends.netcdf.nc_metadata import NCMetadata
 from imas.exception import InvalidNetCDFEntry
 from imas.ids_base import IDSBase
+from imas.ids_convert import NBCPathMap
 from imas.ids_data_type import IDSDataType
 from imas.ids_defs import IDS_TIME_MODE_HOMOGENEOUS
 from imas.ids_metadata import IDSMetadata
@@ -70,19 +71,32 @@ def _tree_iter(
 class NC2IDS:
     """Class responsible for reading an IDS from a NetCDF group."""
 
-    def __init__(self, group: netCDF4.Group, ids: IDSToplevel) -> None:
+    def __init__(
+        self,
+        group: netCDF4.Group,
+        ids: IDSToplevel,
+        ids_metadata: IDSMetadata,
+        nbc_map: Optional[NBCPathMap],
+    ) -> None:
         """Initialize NC2IDS converter.
 
         Args:
             group: NetCDF group that stores the IDS data.
             ids: Corresponding IDS toplevel to store the data in.
+            ids_metadata: Metadata corresponding to the DD version that the data is
+                stored in.
+            nbc_map: Path map for implicit DD conversions.
         """
         self.group = group
         """NetCDF Group that the IDS is stored in."""
         self.ids = ids
         """IDS to store the data in."""
+        self.ids_metadata = ids_metadata
+        """Metadata of the IDS in the DD version that the data is stored in"""
+        self.nbc_map = nbc_map
+        """Path map for implicit DD conversions."""
 
-        self.ncmeta = NCMetadata(ids.metadata)
+        self.ncmeta = NCMetadata(ids_metadata)
         """NetCDF related metadata."""
         self.variables = list(group.variables)
         """List of variable names stored in the netCDF group."""
@@ -114,16 +128,39 @@ def run(self) -> None:
         for var_name in self.variables:
             if var_name.endswith(":shape"):
                 continue
-            metadata = self.ids.metadata[var_name]
+            metadata = self.ids_metadata[var_name]
 
             if metadata.data_type is IDSDataType.STRUCTURE:
                 continue  # This only contains DD metadata we already know
 
+            # Handle implicit DD version conversion
+            if self.nbc_map is None:
+                target_metadata = metadata  # no conversion
+            elif metadata.path_string in self.nbc_map:
+                new_path = self.nbc_map.path[metadata.path_string]
+                if new_path is None:
+                    logging.info(
+                        "Not loading data for %s: no equivalent data structure exists "
+                        "in the target Data Dictionary version.",
+                        metadata.path_string,
+                    )
+                    continue
+                target_metadata = self.ids.metadata[new_path]
+            elif metadata.path_string in self.nbc_map.type_change:
+                logging.info(
+                    "Not loading data for %s: cannot hanlde type changes when "
+                    "implicitly converting data to the target Data Dictionary version.",
+                    metadata.path_string,
+                )
+                continue
+            else:
+                target_metadata = metadata  # no conversion required
+
             var = self.group[var_name]
             if metadata.data_type is IDSDataType.STRUCT_ARRAY:
                 if "sparse" in var.ncattrs():
                     shapes = self.group[var_name + ":shape"][()]
-                    for index, node in tree_iter(self.ids, metadata):
+                    for index, node in tree_iter(self.ids, target_metadata):
                         node.resize(shapes[index][0])
 
                 else:
@@ -132,7 +169,7 @@ def run(self) -> None:
                         metadata.path_string, self.homogeneous_time
                     )[-1]
                     size = self.group.dimensions[dim].size
-                    for _, node in tree_iter(self.ids, metadata):
+                    for _, node in tree_iter(self.ids, target_metadata):
                         node.resize(size)
 
                 continue
@@ -144,22 +181,22 @@ def run(self) -> None:
             if "sparse" in var.ncattrs():
                 if metadata.ndim:
                     shapes = self.group[var_name + ":shape"][()]
-                    for index, node in tree_iter(self.ids, metadata):
+                    for index, node in tree_iter(self.ids, target_metadata):
                         shape = shapes[index]
                         if shape.all():
                             node.value = data[index + tuple(map(slice, shapes[index]))]
                 else:
-                    for index, node in tree_iter(self.ids, metadata):
+                    for index, node in tree_iter(self.ids, target_metadata):
                         value = data[index]
                         if value != getattr(var, "_FillValue", None):
                             node.value = data[index]
 
             elif metadata.path_string not in self.ncmeta.aos:
                 # Shortcut for assigning untensorized data
-                self.ids[metadata.path] = data
+                self.ids[target_metadata.path] = data
 
             else:
-                for index, node in tree_iter(self.ids, metadata):
+                for index, node in tree_iter(self.ids, target_metadata):
                     node.value = data[index]
 
     def validate_variables(self) -> None:
@@ -194,7 +231,7 @@ def validate_variables(self) -> None:
             # Check that the DD defines this variable, and validate its metadata
             var = self.group[var_name]
             try:
-                metadata = self.ids.metadata[var_name]
+                metadata = self.ids_metadata[var_name]
             except KeyError:
                 raise InvalidNetCDFEntry(
                     f"Invalid variable {var_name}: no such variable exists in the "

imas/backends/netcdf/nc_validate.py

@@ -47,8 +47,9 @@ def validate_netcdf_file(filename: str) -> None:
         for ids_name in ids_names:
             for occurrence in entry.list_all_occurrences(ids_name):
                 group = dataset[f"{ids_name}/{occurrence}"]
+                ids = factory.new(ids_name)
                 try:
-                    NC2IDS(group, factory.new(ids_name)).validate_variables()
+                    NC2IDS(group, ids, ids.metadata, None).validate_variables()
                 except InvalidNetCDFEntry as exc:
                     occ = f":{occurrence}" if occurrence else ""
                     raise InvalidNetCDFEntry(f"Invalid IDS {ids_name}{occ}: {exc}")

imas/test/test_nbc_change.py

@@ -9,16 +9,11 @@
 
 import numpy as np
 import pytest
-
 from imas.db_entry import DBEntry
 from imas.ids_convert import convert_ids
 from imas.ids_defs import IDS_TIME_MODE_HOMOGENEOUS, MEMORY_BACKEND
 from imas.ids_factory import IDSFactory
-from imas.test.test_helpers import (
-    compare_children,
-    fill_with_random_data,
-    open_dbentry,
-)
+from imas.test.test_helpers import compare_children, fill_with_random_data, open_dbentry
 
 
 @pytest.fixture(autouse=True)
@@ -97,6 +92,23 @@ def test_nbc_0d_to_1d(caplog, requires_imas):
     entry_339.close()
 
 
+def test_nbc_0d_to_1d_netcdf(caplog, tmp_path):
+    # channel/filter_spectrometer/radiance_calibration in spectrometer visible changed
+    # from FLT_0D to FLT_1D in DD 3.39.0
+    ids = IDSFactory("3.32.0").spectrometer_visible()
+    ids.ids_properties.homogeneous_time = IDS_TIME_MODE_HOMOGENEOUS
+    ids.channel.resize(1)
+    ids.channel[0].filter_spectrometer.radiance_calibration = 1.0
+
+    # Test implicit conversion during get
+    with DBEntry(str(tmp_path / "test.nc"), "x", dd_version="3.32.0") as entry_332:
+        entry_332.put(ids)
+    with DBEntry(str(tmp_path / "test.nc"), "r", dd_version="3.39.0") as entry_339:
+        ids_339 = entry_339.get("spectrometer_visible")  # implicit conversion
+        assert not ids_339.channel[0].filter_spectrometer.radiance_calibration.has_value
+        entry_339.close()
+
+
 def test_nbc_change_aos_renamed():
     """Test renamed AoS in pulse_schedule: ec/antenna -> ec/launcher.
 
@@ -272,7 +284,7 @@ def test_pulse_schedule_aos_renamed_autofill_up(backend, worker_id, tmp_path):
     dbentry.close()
 
 
-def test_pulse_schedule_multi_rename():
+def test_pulse_schedule_multi_rename(tmp_path):
     # Multiple renames of the same element:
     # DD >= 3.40+:  ec/beam
     # DD 3.26-3.40: ec/launcher (but NBC metadata added in 3.28 only)
@@ -294,9 +306,18 @@ def test_pulse_schedule_multi_rename():
     ps["3.40.0"].ec.beam[0].name = name
 
     for version1 in ps:
+        ncfilename = str(tmp_path / f"{version1}.nc")
+        with DBEntry(ncfilename, "x", dd_version=version1) as entry:
+            entry.put(ps[version1])
+
         for version2 in ps:
             converted = convert_ids(ps[version1], version2)
-            compare_children(ps[version2], converted)
+            compare_children(ps[version2].ec, converted.ec)
+
+            # Test with netCDF backend
+            with DBEntry(ncfilename, "r", dd_version=version2) as entry:
+                converted = entry.get("pulse_schedule")
+            compare_children(ps[version2].ec, converted.ec)
 
 
 def test_autofill_save_newer(ids_name, backend, worker_id, tmp_path):

imas/test/test_nc_validation.py

@@ -1,7 +1,6 @@
 import netCDF4
 import numpy as np
 import pytest
-
 from imas.backends.netcdf.ids2nc import IDS2NC
 from imas.backends.netcdf.nc2ids import NC2IDS
 from imas.backends.netcdf.nc_validate import validate_netcdf_file
@@ -32,7 +31,8 @@ def memfile_with_ids(memfile, factory):
     ids.profiles_1d[0].zeff = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]
     IDS2NC(ids, memfile).run()
     # This one is valid:
-    NC2IDS(memfile, factory.core_profiles()).run()
+    ids = factory.core_profiles()
+    NC2IDS(memfile, ids, ids.metadata, None).run()
     return memfile
 
 
@@ -51,66 +51,75 @@ def test_invalid_homogeneous_time(memfile, factory):
 
     ids = factory.core_profiles()
     with pytest.raises(InvalidNetCDFEntry):
-        NC2IDS(empty_group, ids)  # ids_properties.homogeneous_time does not exist
+        # ids_properties.homogeneous_time does not exist
+        NC2IDS(empty_group, ids, ids.metadata, None)
     with pytest.raises(InvalidNetCDFEntry):
-        NC2IDS(invalid_dtype, ids)
+        NC2IDS(invalid_dtype, ids, ids.metadata, None)
     with pytest.raises(InvalidNetCDFEntry):
-        NC2IDS(invalid_shape, ids)
+        NC2IDS(invalid_shape, ids, ids.metadata, None)
     with pytest.raises(InvalidNetCDFEntry):
-        NC2IDS(invalid_value, ids)
+        NC2IDS(invalid_value, ids, ids.metadata, None)
 
 
 def test_invalid_units(memfile_with_ids, factory):
     memfile_with_ids["time"].units = "hours"
+    ids = factory.core_profiles()
     with pytest.raises(InvalidNetCDFEntry):
-        NC2IDS(memfile_with_ids, factory.core_profiles()).run()
+        NC2IDS(memfile_with_ids, ids, ids.metadata, None).run()
 
 
 def test_invalid_documentation(memfile_with_ids, factory, caplog):
+    ids = factory.core_profiles()
     with caplog.at_level("WARNING"):
-        NC2IDS(memfile_with_ids, factory.core_profiles()).run()
+        NC2IDS(memfile_with_ids, ids, ids.metadata, None).run()
     assert not caplog.records
     # Invalid docstring logs a warning
     memfile_with_ids["time"].documentation = "https://en.wikipedia.org/wiki/Time"
     with caplog.at_level("WARNING"):
-        NC2IDS(memfile_with_ids, factory.core_profiles()).run()
+        NC2IDS(memfile_with_ids, ids, ids.metadata, None).run()
     assert len(caplog.records) == 1
 
 
 def test_invalid_dimension_name(memfile_with_ids, factory):
     memfile_with_ids.renameDimension("time", "T")
+    ids = factory.core_profiles()
     with pytest.raises(InvalidNetCDFEntry):
-        NC2IDS(memfile_with_ids, factory.core_profiles()).run()
+        NC2IDS(memfile_with_ids, ids, ids.metadata, None).run()
 
 
 def test_invalid_coordinates(memfile_with_ids, factory):
     memfile_with_ids["profiles_1d.grid.rho_tor_norm"].coordinates = "xyz"
+    ids = factory.core_profiles()
     with pytest.raises(InvalidNetCDFEntry):
-        NC2IDS(memfile_with_ids, factory.core_profiles()).run()
+        NC2IDS(memfile_with_ids, ids, ids.metadata, None).run()
 
 
 def test_invalid_ancillary_variables(memfile_with_ids, factory):
     memfile_with_ids["time"].ancillary_variables = "xyz"
+    ids = factory.core_profiles()
     with pytest.raises(InvalidNetCDFEntry):
-        NC2IDS(memfile_with_ids, factory.core_profiles()).run()
+        NC2IDS(memfile_with_ids, ids, ids.metadata, None).run()
 
 
 def test_extra_attributes(memfile_with_ids, factory):
     memfile_with_ids["time"].new_attribute = [1, 2, 3]
+    ids = factory.core_profiles()
     with pytest.raises(InvalidNetCDFEntry):
-        NC2IDS(memfile_with_ids, factory.core_profiles()).run()
+        NC2IDS(memfile_with_ids, ids, ids.metadata, None).run()
 
 
 def test_shape_array_without_data(memfile_with_ids, factory):
     memfile_with_ids.createVariable("profiles_1d.t_i_average:shape", int, ())
+    ids = factory.core_profiles()
     with pytest.raises(InvalidNetCDFEntry):
-        NC2IDS(memfile_with_ids, factory.core_profiles()).run()
+        NC2IDS(memfile_with_ids, ids, ids.metadata, None).run()
 
 
 def test_shape_array_without_sparse_data(memfile_with_ids, factory):
     memfile_with_ids.createVariable("profiles_1d.grid.rho_tor_norm:shape", int, ())
+    ids = factory.core_profiles()
     with pytest.raises(InvalidNetCDFEntry):
-        NC2IDS(memfile_with_ids, factory.core_profiles()).run()
+        NC2IDS(memfile_with_ids, ids, ids.metadata, None).run()
 
 
 def test_shape_array_with_invalid_dimensions(memfile_with_ids, factory):
@@ -128,7 +137,7 @@ def test_shape_array_with_invalid_dimensions(memfile_with_ids, factory):
         ("time", "profiles_1d.grid.rho_tor_norm:i"),
     )
     with pytest.raises(InvalidNetCDFEntry):
-        NC2IDS(memfile_with_ids, cp).run()
+        NC2IDS(memfile_with_ids, cp, cp.metadata, None).run()
 
 
 def test_shape_array_with_invalid_dtype(memfile_with_ids, factory):
@@ -144,7 +153,7 @@ def test_shape_array_with_invalid_dtype(memfile_with_ids, factory):
         "profiles_1d.t_i_average:shape", float, ("time", "1D")
     )
     with pytest.raises(InvalidNetCDFEntry):
-        NC2IDS(memfile_with_ids, cp).run()
+        NC2IDS(memfile_with_ids, cp, cp.metadata, None).run()
 
 
 def test_validate_nc(tmpdir):