update ci

Author: xadupre

.github/workflows/ci.yml

@@ -18,7 +18,7 @@ jobs:
         os: [ubuntu-latest]
         python: ['3.10', '3.11', '3.12', '3.13']
         transformers: ['4.48.3', '4.51.3', '4.52.4', '4.55.4', '4.56.2', '4.57.1', 'main']
-        torch: ['2.8', '2.9', 'main']
+        torch: ['2.9', 'main']
         exclude:
           - python: '3.10'  # 3.10
             torch: 'main'
@@ -36,8 +36,6 @@ jobs:
             transformers: '4.57.1'
           - python: '3.11'  # 3.11
             torch: 'main'
-          - python: '3.11'
-            torch: '2.8'
           - python: '3.11'
             transformers: 'main'
           - python: '3.11'
@@ -46,9 +44,7 @@ jobs:
             transformers: '4.56.2'
           - python: '3.11'
             transformers: '4.57.1'
-          - python: '3.13'  # 3.13
-            torch: '2.8'
-          - python: '3.13'
+          - python: '3.13'  # 3.11
             torch: '2.9'
           - python: '3.13'
             transformers: '4.48.3'

.github/workflows/model.yml

@@ -0,0 +1,74 @@
+name: ci
+
+on:
+  push:
+  pull_request:
+    types:
+      - closed
+    branches:
+      - main
+
+jobs:
+  run:
+    name: MODEL to-${{ matrix.torch }}-tr-${{ matrix.transformers }}-ci ${{ matrix.os }}-${{ matrix.python }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        python: ['3.12']
+        transformers: ['main']
+        torch: ['main']
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
+
+      - name: Install pytorch ${{ matrix.torch }}
+        run: |
+          if [[ "${{ matrix.torch }}" == "main" ]]; then
+            python -m pip install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cpu
+          else
+            echo "install torch==${{ matrix.torch }} torchvision torchaudio"
+            pip install torch==${{ matrix.torch }} torchvision torchaudio
+          fi
+
+      - name: Install transformers ${{ matrix.transformers }}
+        run: |
+          if [[ "${{ matrix.transformers }}" == "main" ]]; then
+            echo "install transformers from github"
+            git clone https://github.com/huggingface/transformers.git
+            cd transformers
+            pip install -e .
+            cd ..
+          else
+            echo "install transformers==${{ matrix.transformers }}"
+            pip install transformers==${{ matrix.transformers }}
+          fi
+
+      - name: Install requirements
+        run: python -m pip install -r requirements.txt
+
+      - name: Install requirements dev
+        run: python -m pip install -r requirements-dev.txt
+
+      - name: Uninstall onnx-diagnostic
+        run: python -m pip uninstall -y onnx-diagnostic
+
+      - name: Cache pip
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('requirements-dev.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+            ${{ runner.os }}-
+
+      - name: pip freeze
+        run: python -m pip freeze
+
+      - name: qwen2.5
+        run: PYTHONPATH=. UNITTEST_GOING=1 NEVERTEST=1 TESTDTYPE=float16 TESTDEVICE=cpu python _unittests/ut_tasks/try_export.py -f -k test_imagetext2text_qwen_2_5_vl_instruct_visual
+        

_unittests/ut_tasks/try_export.py

@@ -37,13 +37,12 @@ def test_imagetext2text_qwen_2_5_vl_instruct_visual(self):
 
         .. code-block:: bash
 
+            NEVERTEST=1 \\
             QWEN25ATTENTION=BIGMASK \\
             PRETRAINED=1 \\
             TESTDEVICE=cuda \\
             TESTDTYPE=float16 \\
             EXPORTER=custom \\
-            NEVERTEST=1 \\
-            DROPPATTERN=SkipSimplifiedLayerNormalizationMulPattern,SkipSimplifiedLayerNormalizationPattern \\
             python _unittests/ut_tasks/try_export.py -k qwen_2_5_vl_instruct_visual
         """
         device = os.environ.get("TESTDEVICE", "cpu")
@@ -111,7 +110,8 @@ def _config_reduction(config, task):
 
         print(f"-- inputs: {self.string_type(inputs, with_shape=True)}")
         # this is too long
-        expected = model.visual(**inputs)
+        model_to_export = model.visual if hasattr(model, "visual") else model.model.visual
+        expected = model_to_export(**inputs)
         print(f"-- expected: {self.string_type(expected, with_shape=True)}")
 
         filename = self.get_dump_file(
@@ -136,13 +136,13 @@ def _config_reduction(config, task):
             stop_if_static=2,
         ):
             to_onnx(
-                model.visual,
+                model_to_export,
                 kwargs=export_inputs,
                 dynamic_shapes=dynamic_shapes,
                 filename=filename,
                 exporter=exporter,
                 verbose=1,
-                save_ep=(fileep, 2**35),
+                save_ep=None if self.unit_test_going() else (fileep, 2**35),
                 target_opset=22,
                 optimize=True,
                 onnx_plugs=PLUGS,
@@ -159,7 +159,7 @@ def _config_reduction(config, task):
         self.assert_onnx_disc(
             f"test_imagetext2text_qwen_2_5_vl_instruct_visual.{device}.{dtype}.{exporter}",
             filename,
-            model.visual,
+            model_to_export,
             export_inputs,
             verbose=1,
             providers=(
@@ -171,8 +171,11 @@ def _config_reduction(config, task):
             atol=0.02,
             rtol=10,
             ort_optimized_graph=False,
-            ep=pt2_file,
+            # ep=pt2_file,
+            expected=expected,
         )
+        if self.unit_test_going():
+            self.clean_dump()
 
 
 if __name__ == "__main__":

onnx_diagnostic/ext_test_case.py

@@ -64,7 +64,7 @@ def skipif_ci_apple(msg) -> Callable:
     return lambda x: x
 
 
-def unit_test_going():
+def unit_test_going() -> bool:
     """
     Enables a flag telling the script is running while testing it.
     Avois unit tests to be very long.
@@ -743,6 +743,13 @@ class ExtTestCase(unittest.TestCase):
     _warns: List[Tuple[str, int, Warning]] = []
     _todos: List[Tuple[Callable, str]] = []
 
+    def unit_test_going(self):
+        """
+        Enables a flag telling the script is running while testing it.
+        Avois unit tests to be very long.
+        """
+        return unit_test_going()
+
     @property
     def verbose(self):
         "Returns the the value of environment variable ``VERBOSE``."
@@ -1238,7 +1245,7 @@ def assert_onnx_disc(
             if isinstance(proto, str):
                 name = proto
                 proto = onnx.load(name)
-            else:
+            elif not self.unit_tst_going():
                 assert isinstance(
                     proto, onnx.ModelProto
                 ), f"Unexpected type {type(proto)} for proto"
@@ -1309,7 +1316,7 @@ def assert_onnx_disc(
             )
             if verbose:
                 print(f"[{vname}] ep_expected {string_type(ep_expected, **kws)}")
-            ep_diff = max_diff(expected, ep_expected)
+            ep_diff = max_diff(expected, ep_expected, hist=[0.1, 0.01])
             if verbose:
                 print(f"[{vname}] ep_diff {string_diff(ep_diff)}")
             assert (
@@ -1323,11 +1330,11 @@ def assert_onnx_disc(
                 f"discrepancies in {test_name!r} between the model "
                 f"and the exported model diff={string_diff(ep_diff)}"
             )
-            ep_nx_diff = max_diff(ep_expected, got, flatten=True)
+            ep_nx_diff = max_diff(ep_expected, got, flatten=True, hist=[0.1, 0.01])
             if verbose:
                 print(f"[{vname}] ep_nx_diff {string_diff(ep_nx_diff)}")
 
-        diff = max_diff(expected, got, flatten=True)
+        diff = max_diff(expected, got, flatten=True, hist=[0.1, 0.01])
         if verbose:
             print(f"[{vname}] diff {string_diff(diff)}")
         assert (