Merge pull request #260 from astrofrog/skip-hash

Add skip_hash=True/False option to allow hash comparison to be skipped for specific tests

Author: astrofrog

docs/configuration.rst

@@ -207,6 +207,30 @@ If its directory does not exist, it will be created along with any missing paren
 Configuring this option disables baseline image comparison.
 If you want to enable both hash and baseline image comparison, which we call :doc:`"hybrid mode" <hybrid_mode>`, you must explicitly set the :ref:`baseline directory configuration option <baseline-dir>`.
 
+.. _skip-hash:
+
+Skip hash comparison for specific tests
+---------------------------------------
+| **kwarg**: ``skip_hash=<bool>``
+| **CLI**: ---
+| **INI**: ---
+| Default: ``False``
+
+When a global hash library is configured (via CLI or INI), you can disable hash comparison for specific tests by setting ``skip_hash=True``.
+This is useful for tests that have non-deterministic output where exact hash matching is not possible, but baseline image comparison with a tolerance is acceptable.
+
+When ``skip_hash=True`` is set, the test will use baseline image comparison instead of hash comparison, even if a hash library is configured globally.
+
+.. code:: python
+
+    @pytest.mark.mpl_image_compare(skip_hash=True, tolerance=10)
+    def test_plot_with_tolerance():
+        # This test will use baseline image comparison with tolerance,
+        # skipping hash comparison even if --mpl-hash-library is set
+        ...
+
+This option is particularly useful in :doc:`"hybrid mode" <hybrid_mode>` when most tests should use hash comparison for speed and reliability, but a few tests need tolerance-based image comparison due to platform-specific rendering differences.
+
 .. _controlling-sensitivity:
 
 Controlling the sensitivity of the comparison

docs/hybrid_mode.rst

@@ -82,6 +82,24 @@ This is what the basic HTML summary looks like for the same test as above:
 
 :summary:`test_basic_html`
 
+Skipping hash comparison for specific tests
+============================================
+
+In some cases, certain tests may produce images that are not deterministic across platforms or environments.
+For these tests, you can use ``skip_hash=True`` to disable hash comparison and fall back to baseline image comparison with a tolerance:
+
+.. code-block:: python
+
+    @pytest.mark.mpl_image_compare(skip_hash=True, tolerance=10)
+    def test_plot_with_platform_differences():
+        # This test will use baseline image comparison instead of hash comparison
+        fig, ax = plt.subplots()
+        ax.plot([1, 2, 3])
+        return fig
+
+This allows you to use hash comparison for most tests (for speed and reliability), while allowing a few tests to use tolerance-based image comparison.
+See the :ref:`skip_hash configuration option <skip-hash>` for more details.
+
 Continue reading
 ================
 

pytest_mpl/plugin.py

@@ -694,9 +694,11 @@ def save_figure(self, item, fig, filename):
         if deterministic is None:
 
             # The deterministic option should only matter for hash-based tests,
-            # so we first check if a hash library is being used
+            # so we first check if a hash library is being used (and not skipped)
+            skip_hash = compare.kwargs.get('skip_hash', False)
+            use_hash_library = (self.hash_library or compare.kwargs.get('hash_library', None)) and not skip_hash
 
-            if self.hash_library or compare.kwargs.get('hash_library', None):
+            if use_hash_library:
 
                 if ext == 'png':
                     if 'metadata' not in savefig_kwargs or 'Software' not in savefig_kwargs['metadata']:
@@ -910,15 +912,21 @@ def pytest_runtest_call(self, item):  # noqa
                             result_image.relative_to(self.results_dir).as_posix()
 
                 if self.generate_hash_library is not None:
-                    summary['hash_status'] = 'generated'
-                    image_hash = self.generate_image_hash(item, fig)
-                    self._generated_hash_library[test_name] = image_hash
-                    summary['baseline_hash'] = image_hash
+                    skip_hash = compare.kwargs.get('skip_hash', False)
+                    if skip_hash:
+                        summary['hash_status'] = 'skipped'
+                    else:
+                        summary['hash_status'] = 'generated'
+                        image_hash = self.generate_image_hash(item, fig)
+                        self._generated_hash_library[test_name] = image_hash
+                        summary['baseline_hash'] = image_hash
 
                 # Only test figures if not generating images
                 if self.generate_dir is None:
-                    # Compare to hash library
-                    if self.hash_library or compare.kwargs.get('hash_library', None):
+                    # Compare to hash library (skip_hash=True disables for this test)
+                    skip_hash = compare.kwargs.get('skip_hash', False)
+                    use_hash_library = (self.hash_library or compare.kwargs.get('hash_library', None)) and not skip_hash
+                    if use_hash_library:
                         msg = self.compare_image_to_hash_library(item, fig, result_dir, summary=summary)
 
                     # Compare against a baseline if specified

tests/test_hash_library.py

@@ -4,6 +4,93 @@
 from helpers import pytester_path
 
 
+def test_skip_hash(pytester):
+    """Test that skip_hash=True skips hash comparison and uses baseline instead."""
+    path = pytester_path(pytester)
+    hash_library = path / "hash_library.json"
+    baseline_dir = path / "baseline"
+
+    # Generate baseline image (no hash library needed for generation)
+    pytester.makepyfile(
+        """
+        import matplotlib.pyplot as plt
+        import pytest
+        @pytest.mark.mpl_image_compare()
+        def test_mpl():
+            fig = plt.figure()
+            ax = fig.add_subplot(1, 1, 1)
+            ax.plot([1, 3, 2])
+            return fig
+        """
+    )
+    pytester.runpytest(f"--mpl-generate-path={baseline_dir}")
+
+    # Create hash library with bad hash
+    with open(hash_library, "w") as fp:
+        json.dump({"test_skip_hash.test_mpl": "bad-hash-value"}, fp)
+
+    # Without skip_hash: should fail (hash mismatch)
+    result = pytester.runpytest("--mpl",
+                                f"--mpl-hash-library={hash_library}",
+                                f"--mpl-baseline-path={baseline_dir}")
+    result.assert_outcomes(failed=1)
+
+    # With skip_hash=True: should pass (uses baseline comparison, skips hash)
+    pytester.makepyfile(
+        """
+        import matplotlib.pyplot as plt
+        import pytest
+        @pytest.mark.mpl_image_compare(skip_hash=True)
+        def test_mpl():
+            fig = plt.figure()
+            ax = fig.add_subplot(1, 1, 1)
+            ax.plot([1, 3, 2])
+            return fig
+        """
+    )
+    result = pytester.runpytest("--mpl",
+                                f"--mpl-hash-library={hash_library}",
+                                f"--mpl-baseline-path={baseline_dir}")
+    result.assert_outcomes(passed=1)
+
+
+def test_skip_hash_not_generated(pytester):
+    """Test that skip_hash=True tests are not included in generated hash library."""
+    path = pytester_path(pytester)
+    hash_library = path / "hash_library.json"
+
+    pytester.makepyfile(
+        """
+        import matplotlib.pyplot as plt
+        import pytest
+
+        @pytest.mark.mpl_image_compare()
+        def test_normal():
+            fig = plt.figure()
+            ax = fig.add_subplot(1, 1, 1)
+            ax.plot([1, 2, 3])
+            return fig
+
+        @pytest.mark.mpl_image_compare(skip_hash=True)
+        def test_skip():
+            fig = plt.figure()
+            ax = fig.add_subplot(1, 1, 1)
+            ax.plot([3, 2, 1])
+            return fig
+        """
+    )
+    pytester.runpytest(f"--mpl-generate-hash-library={hash_library}")
+
+    # Check generated hash library
+    with open(hash_library) as fp:
+        hashes = json.load(fp)
+
+    # test_normal should be in the hash library
+    assert "test_skip_hash_not_generated.test_normal" in hashes
+    # test_skip should NOT be in the hash library
+    assert "test_skip_hash_not_generated.test_skip" not in hashes
+
+
 @pytest.mark.parametrize(
     "ini, cli, kwarg, success_expected",
     [