Visitar URL original
`InferenceRunner` + `analyze_videos`: Fix top-down analysis crashing for zero-detections and unreadable video handling by deruyter92 · Pull Request #3488 · DeepLabCut/DeepLabCut · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
64a2f4e
fix _generate_output_data crash on empty predictions
deruyter92 Sep 8, 2026
510090e
Allow create_df_from_prediction() to handle an empty predictions list.
deruyter92 Sep 8, 2026
4b45fb9
Treat zero predictions as a distinct warning case in `video_inference()`
deruyter92 Sep 8, 2026
c78f439
Don't write dataframe for empty predictions
deruyter92 Sep 9, 2026
16a2977
update test_videos: pop `"PAFinds"` before comparison
deruyter92 Sep 9, 2026
f534de6
update test_videos: assert no DF is written for empty predictions
deruyter92 Sep 9, 2026
624b1b1
Return empty-detection inference results
C-Achard Sep 9, 2026
c57dd35
Use CTD predictions for video output
C-Achard Sep 9, 2026
4e4dab7
Handle empty prediction outputs gracefully
C-Achard Sep 9, 2026
42098ac
Reuse shared frame key width helper
C-Achard Sep 9, 2026
292d068
Clarify video inference warnings
C-Achard Sep 9, 2026
d5ba290
Go back to ValueError for empty predictions
C-Achard Sep 9, 2026
bff4fea
Remove NaN fallback for empty video predictions
C-Achard Sep 9, 2026
06486de
Fix CTD export guard for empty predictions
C-Achard Sep 9, 2026
e456aeb
Clarify empty CTD prediction handling
C-Achard Sep 9, 2026
a162439
Centralize test fakes
C-Achard Sep 9, 2026
11ff4fd
Add test for async with no images
C-Achard Sep 9, 2026
a210b69
Clarify async inference test comment
C-Achard Sep 9, 2026
c7a8c34
Use shared frame key width helper
C-Achard Sep 9, 2026
8248a10
Merge branch 'dev' into jaap/fix-empty-video-predictions
C-Achard Sep 17, 2026
90849a5
Don't write pickle for unreadable video
deruyter92 Oct 22, 2026
4b8c0ca
Video inference: update warning for empty predictions
deruyter92 Oct 22, 2026
44918dd
InferenceRunner: don't emit results for producer exceptions
deruyter92 Oct 22, 2026
0e40ba8
update code comment
deruyter92 Oct 22, 2026
b0fbd36
fix ShelfWriter: report zero framens when empty
deruyter92 Oct 22, 2026
ce627ae
Skip unreadable videos with shelf writer + improve docstring and log …
deruyter92 Oct 22, 2026
4867b92
Merge branch 'dev' into jaap/fix-empty-video-predictions
deruyter92 Oct 22, 2026
89d06d0
Fix warning typo + update tests
deruyter92 Oct 22, 2026
a67e257
Remove unreachable CTD empty-export branch and reword FMPose error
deruyter92 Oct 22, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 44 additions & 12 deletions deeplabcut/pose_estimation_pytorch/apis/videos.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
TopDownDynamicCropper,
)
from deeplabcut.pose_estimation_pytorch.runners.inference import InferenceConfig
from deeplabcut.pose_estimation_pytorch.runners.shelving import frame_key_width
from deeplabcut.pose_estimation_pytorch.task import Task
from deeplabcut.refine_training_dataset.stitch import stitch_tracklets
from deeplabcut.utils import VideoReader, auxiliaryfunctions
Expand Down Expand Up @@ -228,15 +229,23 @@ def video_inference(
if shelf_writer is not None:
shelf_writer.close()

if shelf_writer is None and len(predictions) != n_frames:
if shelf_writer is None:
tip_url = "https://deeplabcut.github.io/DeepLabCut/docs/recipes/io.html"
header = "#tips-on-video-re-encoding-and-preprocessing"
logging.warning(
f"The video metadata indicates that there {n_frames} in the video, but "
f"only {len(predictions)} were able to be processed. This can happen if "
"the video is corrupted. You can try to fix the issue by re-encoding your "
f"video (tips on how to do that: {tip_url}{header})"
reencoding_tip = (
"This can happen if the video is corrupted. You can try to fix the issue "
f"by re-encoding your video (tips on how to do that: {tip_url}{header})"
)
if len(predictions) == 0:
logging.warning(
f"No predictions were produced for {video.video_path}: the video could not be read. {reencoding_tip}"
)
elif len(predictions) != n_frames:
logging.warning(
f"The video metadata indicates that there are {n_frames} frames in "
f"the video, but only {len(predictions)} were able to be processed. "
f"{reencoding_tip}"
)

return predictions

Expand Down Expand Up @@ -280,6 +289,9 @@ def analyze_videos(
The index of the trained network is specified by parameters in the config file
(in particular the variable 'snapshot_index').

Frames in which no animal is detected are kept in the outputs, with placeholder
values (-1). Videos from which no frame can be read are skipped with a warning.

Args:
config: full path of the config.yaml file for the project
videos: a str (or list of strings) containing the full paths to videos for
Expand Down Expand Up @@ -542,6 +554,7 @@ def analyze_videos(
# Reading video and init variables
videos = collect_video_paths(videos, extensions=video_extensions, shuffle=in_random_order)
h5_files_created = False # Track if any .h5 files were created
skipped_videos = [] # Videos from which no frame could be read

for video in videos:
if destfolder is None:
Expand Down Expand Up @@ -582,6 +595,13 @@ def analyze_videos(
show_gpu_memory=show_gpu_memory,
)
runtime.append(time.time())

num_predicted = len(predictions) if shelf_writer is None else shelf_writer.frames_written
if num_predicted == 0:
logging.warning(f"Skipping {video}: no frames could be read, so no results are written.")
skipped_videos.append(video)
continue
Comment on lines +599 to +603

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could this skip if a 0-frame leftover is present?

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

SHould we remove the shelf in that case?

@deruyter92 deruyter92 Oct 22, 2026 •

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes good suggestion. maybe we should coordinate with #3558.

@Ayskin do you have any remarks on this?


metadata = _generate_metadata(
cfg=loader.project_cfg,
pytorch_config=loader.model_cfg,
Expand Down Expand Up @@ -656,7 +676,7 @@ def analyze_videos(
ctd_predictions.append(dict(bodyparts=pose))

create_df_from_prediction(
predictions=predictions,
predictions=ctd_predictions,
multi_animal=multi_animal,
model_cfg=loader.model_cfg,
dlc_scorer=dlc_scorer,
Expand Down Expand Up @@ -708,6 +728,14 @@ def analyze_videos(
"ensure that the video inference and output generation are correct.\n"
)

if skipped_videos:
logging.warning(
f"{len(skipped_videos)} of {len(videos)} video(s) could not be read; no results were written: "
f"{', '.join(str(v) for v in skipped_videos)}. Re-encoding these videos may "
"help (https://deeplabcut.github.io/DeepLabCut/docs/recipes/io.html"
"#tips-on-video-re-encoding-and-preprocessing)."
)

return dlc_scorer


Expand All @@ -720,10 +748,8 @@ def create_df_from_prediction(
output_prefix: str | Path,
save_as_csv: bool = False,
) -> pd.DataFrame:
pred_bodyparts = np.stack([p["bodyparts"][..., :3] for p in predictions])
pred_unique_bodyparts = None
if len(predictions) > 0 and "unique_bodyparts" in predictions[0]:
pred_unique_bodyparts = np.stack([p["unique_bodyparts"] for p in predictions])
if not predictions:
raise ValueError("Cannot create a results DataFrame from an empty predictions list.")

output_h5 = Path(output_path) / f"{output_prefix}.h5"
output_pkl = Path(output_path) / f"{output_prefix}_full.pickle"
Expand All @@ -733,6 +759,11 @@ def create_df_from_prediction(
individuals = model_cfg["metadata"]["individuals"]
n_individuals = len(individuals)

pred_bodyparts = np.stack([p["bodyparts"][..., :3] for p in predictions])
pred_unique_bodyparts = None
if "unique_bodyparts" in predictions[0]:
pred_unique_bodyparts = np.stack([p["unique_bodyparts"] for p in predictions])

print(f"Saving results in {output_h5} and {output_pkl}")
coords = ["x", "y", "likelihood"]
cols = [[dlc_scorer], bodyparts, coords]
Expand All @@ -743,6 +774,7 @@ def create_df_from_prediction(
cols_names.insert(1, "individuals")

results_df_index = pd.MultiIndex.from_product(cols, names=cols_names)

pred_bodyparts = pred_bodyparts[:, :n_individuals]
df = pd.DataFrame(
pred_bodyparts.reshape((len(pred_bodyparts), -1)),
Expand Down Expand Up @@ -887,7 +919,7 @@ def _generate_output_data(
pose_config: dict,
predictions: list[dict[str, np.ndarray]],
) -> dict:
str_width = int(np.ceil(np.log10(len(predictions))))
str_width = frame_key_width(len(predictions))
output = {
"metadata": {
"nms radius": pose_config.get("nmsradius"),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,9 @@ def _process_batch(
if batch:
_process_batch(batch)

if not predictions_2d:
raise RuntimeError(f"No pose predictions were made for video {video_path}: no frames could be read.")

output_prefix = f"{Path(video_path).stem}_{dlc_scorer}"
output_h5 = dest_folder / f"{output_prefix}.h5"

Expand Down
5 changes: 5 additions & 0 deletions deeplabcut/pose_estimation_pytorch/runners/inference.py
Original file line number Diff line number Diff line change
Expand Up @@ -267,6 +267,11 @@ def _async_inference(
# propagate any exception from the producer immediately
if self._exception is not None:
raise self._exception
# emit images with an available prediction,
# even if no detections were made (which does not produce a queue item)
# unless the producer failed, as its exception is raised below
if self._exception is None:
results.extend(self._extract_results(shelf_writer))

except BaseException as e: # catches KeyboardInterrupt, SystemExit, etc.
# tell producer to quit
Expand Down
31 changes: 26 additions & 5 deletions deeplabcut/pose_estimation_pytorch/runners/shelving.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,23 @@ def __getitem__(self, item: str) -> dict:
return self._db[item]


def frame_key_width(num_frames: int | None, default: int = 5) -> int:
"""Returns the zero-fill width for frame keys in a full-data dict.

Args:
num_frames: The number of frames in the video, if known.
default: The width to use when the number of frames is unknown.

Returns:
The number of leading zeros to pad frame indices with.
"""
if num_frames is None:
return default
if num_frames <= 1:
return 1
return int(np.ceil(np.log10(num_frames)))


class ShelfWriter(ShelfManager):
"""Writes data to a shelf on-the-fly during video analysis.

Expand All @@ -93,9 +110,12 @@ def __init__(self, pose_cfg: dict, filepath: str | Path, num_frames: int | None
self._num_frames = num_frames
self._frame_index = 0

self._str_width = 5
if num_frames is not None:
self._str_width = int(np.ceil(np.log10(num_frames)))
self._str_width = frame_key_width(num_frames)

@property
def frames_written(self) -> int:
"""The number of frames written to the shelf since it was last opened."""
return self._frame_index

def add_prediction(
self,
Expand Down Expand Up @@ -146,8 +166,9 @@ def add_prediction(

def close(self) -> None:
"""Closes the shelf and writes the updated metadata."""
if self._open and self._frame_index > 0:
# Write updated metadata to shelf (top-level indexing required for shelve)
if self._open:
# Write updated metadata to shelf (top-level indexing required for shelve),
# also when no frame was written so readers don't look for missing frames
metadata = self._db["metadata"]
metadata["nframes"] = self._frame_index
self._db["metadata"] = metadata
Expand Down
Loading
Loading