Visitar URL original
Merge pull request #3470 from rysk-t/detector-bugfixes · DeepLabCut/DeepLabCut@b1dd957 · GitHub
Skip to content

Commit b1dd957

Browse files
authored
Merge pull request #3470 from rysk-t/detector-bugfixes
Fix Faster R-CNN detector inference crashing on box_score_thresh: null
2 parents 1150d78 + 2689d30 commit b1dd957

7 files changed

Lines changed: 125 additions & 16 deletions

File tree

‎deeplabcut/core/config/project_config.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -192,6 +192,19 @@ class ProjectConfig(DLCVersionedConfig):
192192
)
193193
croppedtraining: bool | None = None
194194

195+
# TODO @deruyter92 2026-09-01: This should be included as field in the
196+
# config. It requires a version bump of the schema version.
197+
# see https://github.com/DeepLabCut/DeepLabCut/pull/3470
198+
@property
199+
def bboxes_pcutoff(self) -> Fraction:
200+
"""Default confidence cutoff for plotting bounding boxes from detectors.
201+
202+
Note: This property is introduced in v3.1.0 as prospective config
203+
field, but is intentionally not yet defined as such in the schema to
204+
avoid breaking compatibility with older configs.
205+
"""
206+
return 0.6
207+
195208
@property
196209
def bodyparts_list(self) -> list[str]:
197210
# Animal-count agnostic; Always return a list (never "MULTI!", None, etc.)

‎deeplabcut/pose_estimation_pytorch/apis/evaluation.py‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -462,6 +462,7 @@ def evaluate_snapshot(
462462
per_keypoint_evaluation: bool = False,
463463
detector_snapshot: Snapshot | None = None,
464464
pcutoff: float | list[float] | dict[str, float] | None = None,
465+
bboxes_pcutoff: float | None = None,
465466
) -> pd.DataFrame:
466467
"""Evaluates a snapshot. The evaluation results are stored in the .h5 and .csv file
467468
under the subdirectory 'evaluation_results'.
@@ -490,7 +491,12 @@ def evaluate_snapshot(
490491
(if there are any). If a dict is provided, the keys should be bodyparts
491492
mapping to pcutoff values for each bodypart. Bodyparts that are not defined
492493
in the dict will have pcutoff set to 0.6.
494+
bboxes_pcutoff: The cutoff to use for plotting bounding boxes from detectors.
495+
When `None`, the cutoff will be loaded from the project config.
493496
"""
497+
if bboxes_pcutoff is None:
498+
bboxes_pcutoff = cfg.bboxes_pcutoff
499+
494500
head_type = loader.model_cfg["model"]["heads"]["bodypart"]["type"]
495501
if head_type == "DLCRNetHead":
496502
prune_paf_graph.benchmark_paf_graphs(
@@ -622,8 +628,6 @@ def evaluate_snapshot(
622628

623629
df_ground_truth = ensure_multianimal_df_format(loader.df)
624630

625-
bboxes_cutoff = loader.model_cfg.select("detector.model.box_score_thresh") or 0.6
626-
627631
for mode in ["train", "test"]:
628632
df_combined = predictions[mode].merge(df_ground_truth, left_index=True, right_index=True)
629633
bboxes_split = bounding_boxes[mode]
@@ -642,7 +646,7 @@ def evaluate_snapshot(
642646
alpha_value=cfg["alphavalue"],
643647
p_cutoff=cfg["pcutoff"],
644648
bounding_boxes=bboxes_split,
645-
bboxes_cutoff=bboxes_cutoff,
649+
bboxes_cutoff=bboxes_pcutoff,
646650
)
647651

648652
return df_predictions

‎deeplabcut/pose_estimation_pytorch/config/model.py‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
#
1111
"""Model configuration class for DeepLabCut pose estimation models."""
1212

13-
from pydantic import Field
13+
from pydantic import Field, field_validator
1414

1515
from deeplabcut.core.config import DLCBaseConfig
1616
from deeplabcut.core.config.validation import Fraction
@@ -42,10 +42,20 @@ class DetectorModelConfig(DLCBaseConfig):
4242
freeze_bn_stats: Whether to freeze batch normalization statistics
4343
freeze_bn_weights: Whether to freeze batch normalization weights
4444
variant: Specific variant of the detector model
45+
box_score_thresh: The score below which the detector discards proposals
46+
internally. ``null`` is normalized to the default. The value is
47+
deliberately permissive, as raising this value silently drops
48+
detections. Note the distinction with ``bboxes_pcutoff`` in the
49+
project config, which is the cutoff used to *plot* bounding boxes.
4550
"""
4651

4752
type: str = ""
4853
freeze_bn_stats: bool = False
4954
freeze_bn_weights: bool = False
5055
variant: str | None = None
51-
box_score_thresh: Fraction | None = None
56+
box_score_thresh: Fraction = 0.01
57+
58+
@field_validator("box_score_thresh", mode="before")
59+
@classmethod
60+
def normalize_box_score_thresh(cls, v):
61+
return cls.model_fields["box_score_thresh"].default if v is None else v

‎deeplabcut/utils/make_labeled_video.py‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -510,8 +510,8 @@ def create_labeled_video(
510510
When True, f(x) = max(0, (x - pcutoff)/(1 - pcutoff)). Defaults to False.
511511
plot_bboxes (bool, optional): If using Pytorch and in Top-Down mode,
512512
setting this to true will also plot the bounding boxes. Defaults to True.
513-
bboxes_pcutoff (float, optional): If plotting bounding boxes, this overrides the bboxes_pcutoff
514-
set in the model configuration. Defaults to None.
513+
bboxes_pcutoff (float, optional): Minimum bounding box confidence required for plotting.
514+
When ``None``, the value is loaded from the project configuration. Defaults to None.
515515
max_workers (int | None): Maximum number of processes to use for multiprocessing.
516516
Set this parameter to limit the total RAM-usage of simultaneous processes.
517517
Default: no maximum (i.e. number of spawned processes is based on the number of
@@ -585,6 +585,8 @@ def create_labeled_video(
585585
track_method = auxfun_multianimal.get_track_method(cfg, track_method=track_method)
586586
if pcutoff is None:
587587
pcutoff = cfg["pcutoff"]
588+
if bboxes_pcutoff is None:
589+
bboxes_pcutoff = cfg.bboxes_pcutoff
588590

589591
# Get individuals from the config
590592
individuals = cfg.get("individuals", [""])
@@ -604,11 +606,6 @@ def create_labeled_video(
604606
model_config = PoseConfig.from_yaml(model_config_path)
605607
if model_config.select("train_settings.weight_init.memory_replay"):
606608
superanimal_name = model_config["train_settings"]["weight_init"]["dataset"]
607-
if bboxes_pcutoff is None:
608-
bboxes_pcutoff = model_config.select("detector.model.box_score_thresh") or 0.6
609-
else:
610-
if bboxes_pcutoff is None:
611-
bboxes_pcutoff = 0.6
612609

613610
if init_weights == "":
614611
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.get_scorer_name(
@@ -1084,6 +1081,7 @@ def create_video_with_all_detections(
10841081
modelprefix="",
10851082
confidence_to_alpha: bool | Callable[[float], float] = False,
10861083
plot_bboxes: bool = True,
1084+
bboxes_pcutoff: float | None = None,
10871085
**kwargs,
10881086
):
10891087
"""Create a video labeled with all the detections stored in a '*_full.pickle' file.
@@ -1118,6 +1116,8 @@ def create_video_with_all_detections(
11181116
plot_bboxes (bool, optional): If detections were produced using a Pytorch Top-Down model,
11191117
setting this parameter to True will also plot
11201118
the bounding boxes generated by the detector. Defaults to True.
1119+
bboxes_pcutoff (float, optional): Minimum bounding box confidence required for plotting.
1120+
When ``None``, the value is loaded from the project configuration. Defaults to None.
11211121
kwargs (dict, optional): Additional arguments.
11221122
For torch-based shuffles, can be used to specify:
11231123
- snapshot_index
@@ -1127,6 +1127,7 @@ def create_video_with_all_detections(
11271127

11281128
from deeplabcut.core.inferenceutils import Assembler
11291129

1130+
# TODO @deruyter92 2026-09-01: This should be refactored to ProjectConfig.from_any()
11301131
cfg = auxiliaryfunctions.read_config(config)
11311132
trainFraction = cfg["TrainingFraction"][trainingsetindex]
11321133
DLCscorername, _ = auxiliaryfunctions.get_scorer_name(
@@ -1137,6 +1138,9 @@ def create_video_with_all_detections(
11371138
**kwargs,
11381139
)
11391140

1141+
if bboxes_pcutoff is None:
1142+
bboxes_pcutoff = cfg.bboxes_pcutoff
1143+
11401144
videos = collect_video_paths(videos, extensions=video_extensions)
11411145
if not videos:
11421146
return
@@ -1193,9 +1197,6 @@ def create_video_with_all_detections(
11931197
clip = vp(fname=video, sname=outputname, codec="mp4v")
11941198
ny, nx = clip.height, clip.width
11951199

1196-
bboxes_pcutoff = 0.6
1197-
if pytorch_cfg := (metadata.get("data") or {}).get("pytorch-config"):
1198-
bboxes_pcutoff = PoseConfig.from_any(pytorch_cfg).select("detector.model.box_score_thresh") or 0.6
11991200
bboxes_color = (255, 0, 0)
12001201

12011202
for n in trange(clip.nframes):

‎tests/core/config/test_project_config.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -309,3 +309,26 @@ def test_unversioned_yaml_treated_as_v0(self, tmp_path):
309309
config_path.write_text(f"project_path: {tmp_path}\nengine: pytorch\n")
310310
cfg = ProjectConfig.from_yaml(config_path)
311311
assert cfg.config_version == CURRENT_CONFIG_VERSION
312+
313+
314+
# -----------------------------------------------------------------------------
315+
# Fields awaiting the next schema version bump
316+
#
317+
# Each test here describes a field that is exposed as a read-only property today
318+
# because promoting it to a real schema field requires bumping
319+
# CURRENT_CONFIG_VERSION and writing the migration. They are marked strict xfail
320+
# so that the migration landing turns them into failures, as a reminder to drop
321+
# the property and the marker together.
322+
# -----------------------------------------------------------------------------
323+
324+
325+
class TestFieldsPendingSchemaMigration:
326+
@pytest.mark.xfail(
327+
strict=True,
328+
reason=(
329+
"bboxes_pcutoff is a property until the schema version is bumped, "
330+
"see https://github.com/DeepLabCut/DeepLabCut/pull/3470"
331+
),
332+
)
333+
def test_bboxes_pcutoff_is_a_schema_field(self):
334+
assert "bboxes_pcutoff" in ProjectConfig.model_fields

‎tests/pose_estimation_pytorch/config/test_pose_config_creation.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,7 @@ def test_pose_config_creation(case: PoseConfigCase, tmp_path: Path) -> None:
167167
assert detector is not None
168168
assert detector["model"]["type"] == case.detector_type
169169
assert detector["device"] == "auto"
170-
assert detector["model"].get("box_score_thresh") is None
170+
assert detector["model"]["box_score_thresh"] == 0.01
171171
assert cfg["data"]["train"].get("top_down_crop") is not None
172172
else:
173173
assert cfg.get("detector") is None
Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
#
2+
# DeepLabCut Toolbox (deeplabcut.org)
3+
# © A. & M.W. Mathis Labs
4+
# https://github.com/DeepLabCut/DeepLabCut
5+
#
6+
# Please see AUTHORS for contributors.
7+
# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS
8+
#
9+
# Licensed under GNU Lesser General Public License v3.0
10+
#
11+
"""Generated detector configs store ``box_score_thresh: null``; building a
12+
Faster R-CNN from such a config must not leave torchvision's score threshold
13+
as None, which crashes ``postprocess_detections`` during inference."""
14+
15+
import pytest
16+
17+
from deeplabcut.pose_estimation_pytorch.config.model import DetectorModelConfig
18+
from deeplabcut.pose_estimation_pytorch.models.detectors.base import DETECTORS
19+
20+
21+
@pytest.mark.parametrize(
22+
("value", "expected"),
23+
[
24+
(None, 0.01),
25+
(0.0, 0.0),
26+
(0.4, 0.4),
27+
],
28+
)
29+
def test_detector_box_score_thresh(value, expected):
30+
config = DetectorModelConfig(box_score_thresh=value)
31+
32+
assert config.box_score_thresh == expected
33+
34+
35+
def test_detector_box_score_thresh_default():
36+
config = DetectorModelConfig()
37+
38+
assert config.box_score_thresh == 0.01
39+
40+
41+
def test_build_from_config_with_null_threshold():
42+
"""The detector must not be built with torchvision's score_thresh set to None.
43+
44+
Nothing at the model layer normalizes the threshold any more, so this asserts
45+
the property the config layer is relied upon for: a legacy ``null`` never
46+
reaches the torchvision constructor.
47+
"""
48+
config = DetectorModelConfig.from_dict(
49+
{
50+
"type": "FasterRCNN",
51+
"variant": "fasterrcnn_mobilenet_v3_large_fpn",
52+
"box_score_thresh": None,
53+
}
54+
)
55+
56+
detector = DETECTORS.build(config)
57+
58+
assert detector.model.roi_heads.score_thresh == 0.01

0 commit comments

Comments
 (0)