Visitar URL original
Update FMPose3D modelzoo integration by deruyter92 · Pull Request #3221 · DeepLabCut/DeepLabCut · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
8d8795c
Route FMPose3D through modelzoo video inference
deruyter92 Feb 25, 2026
c03fb2f
Move fmpose 3d module to pose_estimation_pytorch/modelzoo
deruyter92 Feb 25, 2026
34909b9
add dedicated FMPose3D video inference function
deruyter92 Feb 25, 2026
6d2eb0d
skip 3d predictions for invalid/empty 2d poses
deruyter92 Feb 25, 2026
785b55f
update tests
deruyter92 Feb 25, 2026
75f160c
Include 3d outputs in return
deruyter92 Feb 25, 2026
afc64a7
Clamp and warn: set max_individuals to 1
deruyter92 Feb 25, 2026
1f4cd90
change 3d data to dataframe
deruyter92 Feb 25, 2026
79306d6
add tests
deruyter92 Feb 25, 2026
04518bd
Update documentation
deruyter92 Feb 25, 2026
de1c573
Update modelzoo documentation fmpose3d
deruyter92 Feb 26, 2026
45f91b2
add commment legenda to __init__ for keypoint refactoring
deruyter92 Feb 26, 2026
a541c84
Add future note for keypoint refactoring
deruyter92 Feb 26, 2026
a96b8ca
fmpose3d: Add pytest markers for fmpose3d, unittest and functional
deruyter92 Feb 26, 2026
dd042b4
Apply formatting
deruyter92 Feb 26, 2026
1b3d94c
fix parameter name change in docs
deruyter92 Feb 26, 2026
8412c73
adjust parameter name -> fmpose_return_3d
deruyter92 Feb 26, 2026
ae8af5a
Update parameter naming superanimal_name
deruyter92 Feb 26, 2026
c0de52e
change recommended superanimal name for fmpose_3d
deruyter92 Feb 28, 2026
b771f33
Merge branch 'main' into jaap/fmpose_modelzoo_integration
deruyter92 Mar 12, 2026
8923015
Merge branch 'main' into jaap/fmpose_modelzoo_integration
deruyter92 Mar 19, 2026
f2c14c5
bump required version extra fmpose3d >= 0.0.8
deruyter92 Mar 19, 2026
9c78ddc
Merge branch 'main' into jaap/fmpose_modelzoo_integration
MMathisLab Mar 19, 2026
18203b9
apply linting
deruyter92 Mar 20, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Apply formatting
  • Loading branch information
deruyter92 committed Feb 26, 2026
commit dd042b41e2cc24094743c99ae597deeb1d3b01de
2 changes: 1 addition & 1 deletion deeplabcut/modelzoo/models_to_framework.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,5 +5,5 @@
"rtmpose_s": "pytorch",
"rtmpose_x": "pytorch",
"fmpose3d_humans": "pytorch",
"fmpose3d_animals": "pytorch"
"fmpose3d_animals": "pytorch",
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
Licensed under GNU Lesser General Public License v3.0
"""

# NOTE: this module may contain items that need refactoring during
# NOTE: this module may contain items that need refactoring during
# the keypoint migration.

# kpt_refactor - Needs attention when refactoring keypoints
Expand Down
22 changes: 6 additions & 16 deletions deeplabcut/pose_estimation_pytorch/modelzoo/fmpose_3d/inference.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,7 @@ def _pose2d_to_dlc_predictions(
all_scores = np.asarray(pose_2d.scores)
if all_kpts.ndim != 4 or all_scores.ndim != 3:
raise ValueError(
"Expected pose_2d keypoints/scores shaped as "
"(num_persons, num_frames, num_bodyparts, {2 or score})."
"Expected pose_2d keypoints/scores shaped as (num_persons, num_frames, num_bodyparts, {2 or score})."
)

num_frames = all_kpts.shape[1]
Expand All @@ -68,9 +67,7 @@ def _pose2d_to_dlc_predictions(
# i/o migration to validated keypoint schemas (parquet)
def _poses3d_to_dataframe(poses_3d: list[np.ndarray], df_2d, scorer_3d: str):
Comment thread
deruyter92 marked this conversation as resolved.
Resolved
"""Create and fill a 3D dataframe using the shared auxiliary helper."""
df_3d, scorer_3d, bodyparts = auxiliaryfunctions_3d.create_empty_df(
df_2d, scorer_3d, "3d"
)
df_3d, scorer_3d, bodyparts = auxiliaryfunctions_3d.create_empty_df(df_2d, scorer_3d, "3d")
n_frames = len(poses_3d)
n_bodyparts = len(bodyparts)
arr = np.full((n_frames, n_bodyparts, 3), np.nan, dtype=float)
Expand Down Expand Up @@ -117,7 +114,7 @@ def _video_inference_fmpose3d(
)
max_individuals = 1

if device is None or device == 'auto':
if device is None or device == "auto":
device = "cuda:0" if torch.cuda.is_available() else "cpu"

if isinstance(video_paths, (str, Path)):
Expand All @@ -136,9 +133,7 @@ def _video_inference_fmpose3d(

api = get_fmpose3d_inference_api(model_type=model_name, device=device)

dest_folder = (
Path(video_paths[0]).parent if dest_folder is None else Path(dest_folder)
)
dest_folder = Path(video_paths[0]).parent if dest_folder is None else Path(dest_folder)
dest_folder.mkdir(parents=True, exist_ok=True)

if create_labeled_video:
Expand All @@ -163,8 +158,7 @@ def _process_batch(frames: list[np.ndarray]) -> None:
num_detected = int(np.asarray(pose_2d.keypoints).shape[0])
if num_detected > 1 and not warned_multi_person_2d:
logger.warning(
"Multiple 2D detections (%s) were found, but FMPose3D 3D lifting "
"uses only the first individual.",
"Multiple 2D detections (%s) were found, but FMPose3D 3D lifting uses only the first individual.",
num_detected,
)
warned_multi_person_2d = True
Expand All @@ -185,7 +179,6 @@ def _process_batch(frames: list[np.ndarray]) -> None:
logger.info("Skipping 3D lifting for batch due to invalid 2D result: %s", e)
all_poses_3d.extend([np.zeros((0, num_bodyparts, 3)) for _ in frames])


batch: list[np.ndarray] = []
for frame in tqdm(video, desc="FMPose3D inference"):
batch.append(frame)
Expand Down Expand Up @@ -225,10 +218,7 @@ def _process_batch(frames: list[np.ndarray]) -> None:
with open(output_json, "w") as f:
json.dump(predictions_2d, f, cls=NumpyEncoder)

poses_3d_serialisable = [
pose.tolist() if isinstance(pose, np.ndarray) else pose
for pose in all_poses_3d
]
poses_3d_serialisable = [pose.tolist() if isinstance(pose, np.ndarray) else pose for pose in all_poses_3d]
output_3d_json = dest_folder / f"{output_prefix}_3d.json"
with open(output_3d_json, "w") as f:
json.dump(
Expand Down