Merge branch 'develop' of https://github.com/iterorganization/imas-python into feature/lazy-load-from-netcdf

Author: maarten-ic

.github/workflows/linting.yml

@@ -5,7 +5,7 @@ on: push
 jobs:
   build:
 
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     steps:
       - name: Checkout IMAS-Python sources
@@ -14,7 +14,9 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.x'
+          # until saxonche is available in 3.13
+          # https://saxonica.plan.io/issues/6561
+          python-version: "<3.13"
 
       - name: Display Python version
         run: python -c "import sys; print(sys.version)"

.github/workflows/publish.yml

@@ -5,15 +5,17 @@ on: push
 jobs:
   build:
     name: Build distribution
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
     - uses: actions/checkout@v4
       with:
         fetch-depth: 0 
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: "3.x"
+        # until saxonche is available in 3.13
+        # https://saxonica.plan.io/issues/6561
+        python-version: "<3.13"
     - name: Install pypa/build
       run: >-
         python3 -m pip install pip setuptools wheel build
@@ -30,7 +32,7 @@ jobs:
     if: startsWith(github.ref, 'refs/tags/')  # only publish to PyPI on tag pushes
     needs:
     - build
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     environment:
       name: pypi
       url: https://pypi.org/p/imas-python
@@ -50,7 +52,7 @@ jobs:
     if: github.ref=='refs/heads/develop'  # only publish to TestPyPI on develop pushes
     needs:
     - build
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     environment:
       name: testpypi
       url: https://test.pypi.org/p/imas-python

ci/run_pytest.sh

@@ -13,7 +13,7 @@ source /etc/profile.d/modules.sh
 module purge
 # Modules are supplied as arguments in the CI job:
 if [ -z "$@" ]; then
-    module load IMAS-AL-Core
+    module load IMAS-AL-Core Java MDSplus 
 else
     module load $@
 fi

imas/backends/imas_core/mdsplus_model.py

@@ -1,7 +1,6 @@
 # Helper functions to create MDSPlus reference models
 # and store them in a cache directory (.cache/imas/MDSPlus/name-HASH/)
-"""Module for generating and working with MDSplus models.
-"""
+"""Module for generating and working with MDSplus models."""
 
 import errno
 import getpass
@@ -235,6 +234,14 @@ def model_exists(path: Path) -> bool:
     )
 
 
+def transform_with_xslt(xslt_processor, source, xslfile, output_file):
+    return xslt_processor.transform_to_file(
+        source_file=str(source),
+        stylesheet_file=str(xslfile),
+        output_file=str(output_file),
+    )
+
+
 def create_model_ids_xml(cache_dir_path, fname, version):
     """Use Saxon/C to compile an ids.xml suitable for creating an MDSplus model."""
     try:
@@ -243,40 +250,32 @@ def create_model_ids_xml(cache_dir_path, fname, version):
 
             with PySaxonProcessor(license=False) as proc:
                 xslt_processor = proc.new_xslt30_processor()
-
-                xslt_processor.compile_stylesheet(stylesheet_file=str(xslfile))
-
-                input_xml = get_dd_xml(version) if version else None
-                if fname:
-                    source_file = str(fname)
-                elif input_xml:
-                    source_file = input_xml  # Use standard input for the XML string
-                else:
-                    raise ValueError(
-                        "Either 'fname' or 'version' must be provided to generate XML."
-                    )
-
-                # xdm_ddgit = proc.make_string_value(str(version or fname))
-                # xsltproc.set_parameter("DD_GIT_DESCRIBE", xdm_ddgit)
-                # xdm_algit = proc.make_string_value(os.environ.get
-                # ("AL_VERSION", "0.0.0"))
-                # xsltproc.set_parameter("AL_GIT_DESCRIBE", xdm_algit)
-                # Transform XML
-                result = xslt_processor.transform_to_file(
-                    source_file=source_file,
-                    output_file=str(output_file),
-                    initial_template_params={
-                        "DD_GIT_DESCRIBE": str(version or fname),
-                        "AL_GIT_DESCRIBE": os.environ.get("AL_VERSION", "0.0.0"),
-                    },
+                xdm_ddgit = proc.make_string_value(str(version) or fname)
+                xslt_processor.set_parameter("DD_GIT_DESCRIBE", xdm_ddgit)
+                xdm_algit = proc.make_string_value(
+                    os.environ.get("AL_VERSION", "0.0.0")
                 )
-
-                if result is False:
-                    logger.error(
-                        "Transformation failed: Check Saxon/C logs for details."
-                    )
-                    raise RuntimeError("Saxon/C XSLT transformation failed.")
-
+                xslt_processor.set_parameter("AL_GIT_DESCRIBE", xdm_algit)
+                if (
+                    fname is not None
+                    and fname != "-"
+                    and fname != ""
+                    and os.path.exists(fname)
+                ):
+                    transform_with_xslt(xslt_processor, fname, xslfile, output_file)
+                elif version is not None and version != "":
+                    xml_string = get_dd_xml(version)
+
+                    with tempfile.NamedTemporaryFile(
+                        delete=True, mode="w+b"
+                    ) as temp_file:
+                        temp_file.write(xml_string)
+                        temp_file.seek(0)
+                        transform_with_xslt(
+                            xslt_processor, temp_file.name, xslfile, output_file
+                        )
+                else:
+                    raise MDSPlusModelError("Either fname or version must be provided")
     except Exception as e:
         if fname:
             logger.error("Error making MDSplus model IDS.xml for %s", fname)

pyproject.toml

@@ -71,7 +71,8 @@ dependencies = [
 [project.optional-dependencies]
 # these self-dependencies are available since pip 21.2
 all = [
-    "imas[test,docs,imas-core,netcdf,h5py]"
+    "imas-python[test,docs,netcdf,h5py]" 
+    # "imas-python[test,docs,imas-core,netcdf,h5py]" TODO enable when imas-core is available on pypi
 ]
 docs = [
     "sphinx>=6.0.0,<7.0.0",