From 6480005bbb5c53cf1b888d5656201abf90222f3f Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Thu, 31 Jul 2025 03:30:21 -0700 Subject: [PATCH 1/4] allow skip and xfail to both be applied --- mypy/test/data.py | 12 ++++++------ test-data/unit/check-skipfail.test | 20 ++++++++++++++++++++ 2 files changed, 26 insertions(+), 6 deletions(-) create mode 100644 test-data/unit/check-skipfail.test diff --git a/mypy/test/data.py b/mypy/test/data.py index 5b0ad84c0ba77..9f3bf23528bb8 100644 --- a/mypy/test/data.py +++ b/mypy/test/data.py @@ -312,9 +312,10 @@ def __init__( def runtest(self) -> None: if self.skip: pytest.skip() - # TODO: add a better error message for when someone uses skip and xfail at the same time - elif self.xfail: + if self.xfail: self.add_marker(pytest.mark.xfail) + if self.skip and self.xfail: + print(f"The test {self.name} is marked as both skip and xfail, which is bad, because I can't do both at once.", file=sys.stderr) parent = self.getparent(DataSuiteCollector) assert parent is not None, "Should not happen" suite = parent.obj() @@ -659,8 +660,7 @@ def pytest_pycollect_makeitem(collector: Any, name: str, obj: object) -> Any | N r"(?P-only_when_cache|-only_when_nocache)?" r"(?P-skip_path_normalization)?" r"(-(?Pposix|windows))?" - r"(?P-skip)?" - r"(?P-xfail)?" + r"(?:(?P-skip)?(?P-xfail)?|(?P-xfail)?(?P-skip)?)?" ) @@ -699,8 +699,8 @@ def split_test_cases( writescache=bool(m.group("writescache")), only_when=m.group("only_when"), platform=m.group("platform"), - skip=bool(m.group("skip")), - xfail=bool(m.group("xfail")), + skip=bool(m.group("skip") or m.group("skip2")), + xfail=bool(m.group("xfail") or m.group("xfail2")), normalize_output=not m.group("skip_path_normalization"), data=data, line=line_no, diff --git a/test-data/unit/check-skipfail.test b/test-data/unit/check-skipfail.test new file mode 100644 index 0000000000000..b45bf3c66537b --- /dev/null +++ b/test-data/unit/check-skipfail.test @@ -0,0 +1,20 @@ +-- Because this file tests both "-skip" and "-xfail" themselves its results are not very meaningful, except that they should not fail. +-- If you ever want to stop getting s and x results for these, you will have to figure out how to test these features instead. + +-- What does it mean to both skip and xfail a file? Well, since the two are semantically different, according to https://docs.pytest.org/en/stable/how-to/skipping.html, there may be a meaningful answer to that question. For instance, "skipping tests that depend on an external resource which is not available at the moment", "for a feature not yet implemented". Or, as another example, if a feature hasn't been completed, but the test for it is also taking an enormous amount of time, you might xfail it and also skip. +-- You won't be able to tell from the test cases below, but if skip and xfail are both applied, the text case is skipped, not xfailed (which makes sense). The order does not matter. + +[case skipfail1-skip-xfail] +x: int = 1 + +[case skipfail2-xfail-skip] +x: int = 1 + +[case skipfail0] +x: int = 1 + +[case skipfail3-skip] +x: int = 1 + +[case skipfail4-xfail] +x: int = "1" From d045bdb6714c35241be290ddb04a85b2a0ddff08 Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Thu, 31 Jul 2025 03:31:45 -0700 Subject: [PATCH 2/4] it's not bad --- mypy/test/data.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/mypy/test/data.py b/mypy/test/data.py index 9f3bf23528bb8..251e79c089cee 100644 --- a/mypy/test/data.py +++ b/mypy/test/data.py @@ -314,8 +314,6 @@ def runtest(self) -> None: pytest.skip() if self.xfail: self.add_marker(pytest.mark.xfail) - if self.skip and self.xfail: - print(f"The test {self.name} is marked as both skip and xfail, which is bad, because I can't do both at once.", file=sys.stderr) parent = self.getparent(DataSuiteCollector) assert parent is not None, "Should not happen" suite = parent.obj() From 3a564d7b3a0224d223e813186ae0df4331ba3e05 Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Thu, 31 Jul 2025 06:38:51 -0400 Subject: [PATCH 3/4] Apply suggestions from my code review: my typos --- test-data/unit/check-skipfail.test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test-data/unit/check-skipfail.test b/test-data/unit/check-skipfail.test index b45bf3c66537b..413e8709d0a51 100644 --- a/test-data/unit/check-skipfail.test +++ b/test-data/unit/check-skipfail.test @@ -1,8 +1,8 @@ -- Because this file tests both "-skip" and "-xfail" themselves its results are not very meaningful, except that they should not fail. -- If you ever want to stop getting s and x results for these, you will have to figure out how to test these features instead. --- What does it mean to both skip and xfail a file? Well, since the two are semantically different, according to https://docs.pytest.org/en/stable/how-to/skipping.html, there may be a meaningful answer to that question. For instance, "skipping tests that depend on an external resource which is not available at the moment", "for a feature not yet implemented". Or, as another example, if a feature hasn't been completed, but the test for it is also taking an enormous amount of time, you might xfail it and also skip. --- You won't be able to tell from the test cases below, but if skip and xfail are both applied, the text case is skipped, not xfailed (which makes sense). The order does not matter. +-- What does it mean to both skip and xfail a test? Well, since the two are semantically different, according to https://docs.pytest.org/en/stable/how-to/skipping.html, there may be a meaningful answer to that question. For instance, "skipping tests that depend on an external resource which is not available at the moment", "for a feature not yet implemented". Or, as another example, if a feature hasn't been completed, but the test for it is also taking an enormous amount of time, you might xfail it and also skip it. +-- You won't be able to tell from the test cases below, but if skip and xfail are both applied, the test case is skipped, not xfailed (which makes sense). The order does not matter. [case skipfail1-skip-xfail] x: int = 1 From f10fc86e4ddeba0f4b3fb6a6c10671e771e69335 Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Fri, 3 Apr 2026 22:27:21 -0500 Subject: [PATCH 4/4] document where these tests will probably eventually end up --- test-data/unit/check-skipfail.test | 1 + 1 file changed, 1 insertion(+) diff --git a/test-data/unit/check-skipfail.test b/test-data/unit/check-skipfail.test index 413e8709d0a51..f7581d464a919 100644 --- a/test-data/unit/check-skipfail.test +++ b/test-data/unit/check-skipfail.test @@ -1,5 +1,6 @@ -- Because this file tests both "-skip" and "-xfail" themselves its results are not very meaningful, except that they should not fail. -- If you ever want to stop getting s and x results for these, you will have to figure out how to test these features instead. +-- Perhaps by moving them to mypy/test/meta/test_parse_data.py -- What does it mean to both skip and xfail a test? Well, since the two are semantically different, according to https://docs.pytest.org/en/stable/how-to/skipping.html, there may be a meaningful answer to that question. For instance, "skipping tests that depend on an external resource which is not available at the moment", "for a feature not yet implemented". Or, as another example, if a feature hasn't been completed, but the test for it is also taking an enormous amount of time, you might xfail it and also skip it. -- You won't be able to tell from the test cases below, but if skip and xfail are both applied, the test case is skipped, not xfailed (which makes sense). The order does not matter.