|
13 | 13 | import pickle |
14 | 14 | import urllib.request |
15 | 15 | import zipfile |
| 16 | +from enum import Enum, unique |
16 | 17 | from io import BytesIO |
| 18 | +from typing import NamedTuple |
17 | 19 |
|
18 | 20 | import numpy as np |
19 | 21 | import pytest |
| 22 | +from packaging.version import Version |
20 | 23 | from PIL import Image |
21 | 24 | from tqdm import tqdm |
22 | 25 |
|
|
26 | 29 | configure_pandas_future_if_enabled() |
27 | 30 |
|
28 | 31 | from deeplabcut.core import inferenceutils # noqa: E402 |
| 32 | +from deeplabcut.version import __version__ as _dlc_version # noqa: E402 |
29 | 33 |
|
30 | 34 | TESTS_DIR = os.path.dirname(os.path.realpath(__file__)) |
31 | 35 | TEST_DATA_DIR = os.path.join(TESTS_DIR, "data") |
@@ -192,3 +196,69 @@ def read_frame(self, crop=False): |
192 | 196 | def fake_video_reader(): |
193 | 197 | """Factory for synthetic videos: ``fake_video_reader(nframes, fps=..., width=..., height=...)``.""" |
194 | 198 | return FakeVideoReader |
| 199 | + |
| 200 | + |
| 201 | +# ----------------------------------------------------------------------------- |
| 202 | +# Known defects |
| 203 | +# |
| 204 | +# Mark a test that captures a known defect with |
| 205 | +# ``@pytest.mark.known_defect("NAME")``. |
| 206 | +# |
| 207 | +# Through the defect's ``affects_through`` version, the test is treated as a |
| 208 | +# strict xfail. Only an AssertionError is considered an expected failure; |
| 209 | +# setup, import, and other errors still fail the test. |
| 210 | +# |
| 211 | +# After ``affects_through``, the xfail is disabled and the test result is |
| 212 | +# determined by its assertions. Each marked test also emits a PytestWarning |
| 213 | +# indicating that the marker should be removed or assigned a later version. |
| 214 | +# |
| 215 | +# If the defect is fixed before or during the affected version range, the test |
| 216 | +# produces XPASS(strict). Remove the registry entry and all markers that |
| 217 | +# reference it once the defect is fixed. |
| 218 | +# ----------------------------------------------------------------------------- |
| 219 | + |
| 220 | + |
| 221 | +class KnownDefectInfo(NamedTuple): |
| 222 | + affects_through: Version # last DeepLabCut version known to carry the defect |
| 223 | + reason: str |
| 224 | + |
| 225 | + |
| 226 | +@unique |
| 227 | +class KnownDefect(Enum): |
| 228 | + """Known defects pinned by strict-xfail tests, one entry per topic.""" |
| 229 | + |
| 230 | + |
| 231 | +_DLC_VERSION = Version(_dlc_version) |
| 232 | + |
| 233 | + |
| 234 | +def pytest_configure(config: pytest.Config) -> None: |
| 235 | + config.addinivalue_line( |
| 236 | + "markers", |
| 237 | + "known_defect(name): strict xfail for the KnownDefect entry `name`, see tests/conftest.py", |
| 238 | + ) |
| 239 | + |
| 240 | + |
| 241 | +def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: |
| 242 | + for item in items: |
| 243 | + for marker in item.iter_markers("known_defect"): |
| 244 | + (name,) = marker.args |
| 245 | + try: |
| 246 | + defect = KnownDefect[name].value |
| 247 | + except KeyError: |
| 248 | + raise pytest.UsageError(f"{item.nodeid}: unknown KnownDefect {name!r}") from None |
| 249 | + if _DLC_VERSION > defect.affects_through: |
| 250 | + item.warn( |
| 251 | + pytest.PytestWarning( |
| 252 | + f"KnownDefect.{name} is past {defect.affects_through}, so known_defect no longer applies. " |
| 253 | + "If this test passes, delete the marker and the entry; " |
| 254 | + "if it fails, fix the defect or move affects_through forward." |
| 255 | + ) |
| 256 | + ) |
| 257 | + item.add_marker( |
| 258 | + pytest.mark.xfail( |
| 259 | + _DLC_VERSION <= defect.affects_through, |
| 260 | + reason=f"{name} (known through {defect.affects_through}): {defect.reason}", |
| 261 | + raises=AssertionError, |
| 262 | + strict=True, |
| 263 | + ) |
| 264 | + ) |
0 commit comments