Visitar URL original
SuperAnimal Model Updates by n-poulsen · Pull Request #2756 · DeepLabCut/DeepLabCut · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
55 commits
Select commit Hold shift + click to select a range
d6ee416
refactored weight init for better modularity
Sep 18, 2024
59511e3
refactored model downloading
Oct 1, 2024
a0fdeaf
update examples
Oct 1, 2024
58bf846
removed unused code
Oct 1, 2024
23b66f5
various bug fixes
Oct 1, 2024
2764b38
do not create pose_cfg for pytorch models
Oct 1, 2024
881f174
bug fix: keypoint matching
Oct 1, 2024
718d61b
bug fix: keypoint matching
Oct 1, 2024
f1422e1
fixed dlc scorer for zeroshot video
Oct 2, 2024
8f8510f
bug fix - super animal scorer
Oct 2, 2024
44613ab
fix max individuals when updating config
Oct 2, 2024
3a7b161
better check for annotation files
Oct 2, 2024
2a4c6a3
small improvements
Oct 2, 2024
02a279d
update SA detector augmentation
Oct 2, 2024
5c0656f
added SuperBird configs
Oct 2, 2024
7df067f
bug fix: config creation, set method
Oct 2, 2024
8fc6950
bug fixes: memory replay training
Oct 2, 2024
431e913
bug fix: evaluating with memory replay
Oct 2, 2024
04fa197
fix video analysis with memory replay
Oct 2, 2024
42d1081
added SuperBird colormap
Oct 2, 2024
c42f474
added SuperBird to pseudo-label
Oct 2, 2024
819d0e2
raise error when training with memory replay without detector
Oct 2, 2024
da13693
bug fix: raising error when no conversion table
Oct 2, 2024
d939e3c
fix pose config creation - joints for test config
Oct 3, 2024
b0b2208
bug fix
Oct 3, 2024
657b704
add default BU augmentation when Fine-Tuning SA model without detector
Oct 3, 2024
2e6c120
bug fix
Oct 3, 2024
3f87fee
clean video inference. add individuals to metadata.
Oct 3, 2024
e0c8302
bug fix: test pose cfg creation
Oct 3, 2024
a0bff4e
fix make labeled video with memory replay
Oct 3, 2024
fe00587
clean weight init
Oct 3, 2024
d54e7e1
bug fix
Oct 3, 2024
1177ef8
doc improvements and fixes to project configs
Oct 3, 2024
978845b
doc improvements
Oct 3, 2024
e588712
remove extra FIXME
Oct 3, 2024
0221957
doc fixes
Oct 3, 2024
0d008df
fix setup.py
Oct 3, 2024
16e4154
updated SuperAnimal COLAB
Oct 3, 2024
8b18244
updated docs
Oct 3, 2024
f62f1bd
update DLCLibrary version
Oct 3, 2024
26ae7db
fix modelzoo GUI
Oct 3, 2024
714d383
bug fix: tensorflow superanimal video creation
Oct 3, 2024
1b44363
fix cannot pickle memoryview, only use multianimalproject key when an…
Oct 7, 2024
fcab1ce
add cropping argument to SuperAnimal inference
Oct 7, 2024
ad3f7f8
add make_pytorch_test_config to config __init__
Oct 7, 2024
e94ce12
clean superbird config
Oct 7, 2024
480a2ac
do not pin detector dataloader memory
Oct 8, 2024
f18a02f
update SuperAnimal resnet_50 config
Oct 9, 2024
bdab2cd
add mobilenet fasterrcnn superanimal config
Oct 9, 2024
61e9ef6
bug fix: make_labeled_video SA finetune with mem replay, single anima…
Oct 9, 2024
d83503f
fix SuperAnimal memory replay evaluation
Oct 9, 2024
9e04427
update code for new models
Oct 10, 2024
4655b62
build_weight_init with str/path instead of loaded config
Oct 10, 2024
6b0883d
update dlclibrary to use get_available_datasets
Oct 15, 2024
0c6b4e4
updated config when loading SuperAnimal config
Oct 18, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
various bug fixes
  • Loading branch information
Niels Poulsen
Niels Poulsen committed Oct 1, 2024
commit 23b66f533e6749d53c2e19e8440df3009c3e49a4
2 changes: 1 addition & 1 deletion deeplabcut/core/weight_init.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ def to_dict(self) -> dict:
data["dataset"] = self.dataset

if self.detector_snapshot_path is not None:
data["detector_snapshot_path"] = self.detector_snapshot_path
data["detector_snapshot_path"] = str(self.detector_snapshot_path)

if self.conversion_array is not None:
data["conversion_array"] = self.conversion_array.tolist()
Expand Down
4 changes: 3 additions & 1 deletion deeplabcut/gui/displays/selected_shuffle_display.py
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,9 @@ def _update_display(self, new_index: int) -> None:
return

if not pose_cfg_path.exists():
self._set_text_error()
self._set_text_error(
f"The model configuration file {pose_cfg_path} was not created"
)
return

self._read_pose_config(pose_cfg_path)
Expand Down
55 changes: 38 additions & 17 deletions deeplabcut/gui/tabs/create_training_dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import os
from pathlib import Path

import dlclibrary
from PySide6 import QtWidgets
from PySide6.QtCore import Qt, Slot
from PySide6.QtGui import QIcon
Expand Down Expand Up @@ -441,7 +442,10 @@ def update_detectors(
self.detector_choice.removeItem(0)

self.detector_choice.addItems(detectors)
if "ssdlite" in detectors:
default_detector = self.get_default_detector()
if default_detector in detectors:
self.detector_choice.setCurrentIndex(detectors.index(default_detector))
elif "ssdlite" in detectors:
self.detector_choice.setCurrentIndex(detectors.index("ssdlite"))

if net_choice is None:
Expand Down Expand Up @@ -483,7 +487,10 @@ def get_net_filter(self) -> list[str] | None:
return None

weight_init_cfg = _WEIGHT_INIT_OPTIONS[self.weight_init_selector.weight_init]
return weight_init_cfg["model_filter"]
if "super_animal" in weight_init_cfg:
return dlclibrary.get_available_models(weight_init_cfg["super_animal"])

return None

def get_detector_filter(self) -> list[str] | None:
"""Returns: the detectors that can be used based on weight initialization"""
Expand All @@ -494,7 +501,10 @@ def get_detector_filter(self) -> list[str] | None:
return None

weight_init_cfg = _WEIGHT_INIT_OPTIONS[self.weight_init_selector.weight_init]
return weight_init_cfg["detector_filter"]
if "super_animal" in weight_init_cfg:
return dlclibrary.get_available_detectors(weight_init_cfg["super_animal"])

return None

def get_default_net(self) -> str | None:
"""Returns: the net type that can be used based on weight initialization"""
Expand All @@ -507,6 +517,17 @@ def get_default_net(self) -> str | None:
weight_init_cfg = _WEIGHT_INIT_OPTIONS[self.weight_init_selector.weight_init]
return weight_init_cfg.get("default_net")

def get_default_detector(self) -> str | None:
"""Returns: the detector type that can be used based on weight initialization"""
if self.root.engine != Engine.PYTORCH:
return None

if self.weight_init_selector.weight_init not in _WEIGHT_INIT_OPTIONS:
return None

weight_init_cfg = _WEIGHT_INIT_OPTIONS[self.weight_init_selector.weight_init]
return weight_init_cfg.get("default_detector")

def view_shuffles(self) -> None:
viewer = ShuffleMetadataViewer(root=self.root, parent=self)
viewer.show()
Expand Down Expand Up @@ -712,34 +733,34 @@ def _create_confirmation_box(title, description):
"model_filter": None,
"detector_filter": None,
},
"Transfer Learning - SuperAnimal Bird": {
"default_net": "top_down_resnet_50",
"default_detector": "ssdlite",
"super_animal": "superanimal_bird",
},
"Transfer Learning - SuperAnimal Quadruped": {
"default_net": "top_down_hrnet_w32",
"model_filter": [
"dekr_w32",
"hrnet_w32",
],
"detector_filter": ["fasterrcnn_resnet50_fpn_v2"],
"default_detector": "fasterrcnn_resnet50_fpn_v2",
"super_animal": "superanimal_quadruped",
},
"Transfer Learning - SuperAnimal TopViewMouse": {
"default_net": "top_down_hrnet_w32",
"model_filter": [
"dekr_w32",
"hrnet_w32",
],
"detector_filter": ["fasterrcnn_resnet50_fpn_v2"],
"default_detector": "fasterrcnn_resnet50_fpn_v2",
"super_animal": "superanimal_topviewmouse",
},
"Fine-tuning - SuperAnimal Bird": {
"default_net": "top_down_resnet_50",
"default_detector": "ssdlite",
"super_animal": "superanimal_bird",
},
"Fine-tuning - SuperAnimal Quadruped": {
"default_net": "top_down_hrnet_w32",
"model_filter": ["hrnet_w32"], # FIXME - Add ResNet
"detector_filter": ["fasterrcnn_resnet50_fpn_v2"],
"default_detector": "fasterrcnn_resnet50_fpn_v2",
"super_animal": "superanimal_quadruped",
},
"Fine-tuning - SuperAnimal TopViewMouse": {
"default_net": "top_down_hrnet_w32",
"model_filter": ["hrnet_w32"], # FIXME - Add ResNet
"detector_filter": ["fasterrcnn_resnet50_fpn_v2"],
"default_detector": "fasterrcnn_resnet50_fpn_v2",
"super_animal": "superanimal_topviewmouse",
},
}
4 changes: 2 additions & 2 deletions deeplabcut/modelzoo/weight_initialization.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ def build_weight_init(
cfg: dict,
super_animal: str,
model_name: str,
detector_name: str,
detector_name: str | None,
with_decoder: bool = False,
memory_replay: bool = False,
customized_pose_checkpoint: str | Path | None = None,
Expand Down Expand Up @@ -84,7 +84,7 @@ def build_weight_init(
)

detector_snapshot_path = customized_detector_checkpoint
if detector_snapshot_path is None:
if detector_snapshot_path is None and detector_name is not None:
detector_snapshot_path = get_super_animal_snapshot_path(
dataset=super_animal,
model_name=detector_name,
Expand Down
23 changes: 15 additions & 8 deletions deeplabcut/pose_estimation_pytorch/modelzoo/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,15 @@
get_super_animal_model_config_path,
get_super_animal_project_config_path,
)
from deeplabcut.pose_estimation_pytorch.task import Task


def make_super_animal_finetune_config(
weight_init: WeightInitialization,
project_config: dict,
pose_config_path: str,
model_name: str | None = None,
detector_name: str | None = None,
model_name: str,
detector_name: str | None,
) -> dict:
"""
Creates a PyTorch pose configuration file to finetune a SuperAnimal model on a
Expand All @@ -42,7 +43,8 @@ def make_super_animal_finetune_config(
project_config: The project configuration.
pose_config_path: The path where the pose configuration file will be saved
model_name: The type of neural net to finetune.
detector_name: The type of detector to use for the SuperAnimal model.
detector_name: The type of detector to use for the SuperAnimal model. If None is
given, the model will be set to a Bottom-Up framework.

Returns:
The generated pose configuration file.
Expand Down Expand Up @@ -98,7 +100,7 @@ def make_super_animal_finetune_config(
def create_config_from_modelzoo(
super_animal: str,
model_name: str,
detector_name: str,
detector_name: str | None,
converted_bodyparts: list[str],
weight_init: WeightInitialization,
project_config: dict,
Expand All @@ -109,7 +111,8 @@ def create_config_from_modelzoo(
Args:
super_animal: The SuperAnimal dataset on which the model was trained.
model_name: The type of neural net to finetune.
detector_name: The type of detector to use for the SuperAnimal model.
detector_name: The type of detector to use for the SuperAnimal model. If None is
given, the model will be set to a Bottom-Up framework.
converted_bodyparts: The project bodyparts that the model will learn.
weight_init: The weight initialization to use.
project_config: The project configuration.
Expand All @@ -122,9 +125,13 @@ def create_config_from_modelzoo(
model_cfg = config_utils.read_config_as_dict(
get_super_animal_model_config_path(model_name)
)
model_cfg["detector"] = config_utils.read_config_as_dict(
get_super_animal_model_config_path(detector_name)
)
if detector_name is None:
model_cfg["method"] = str(Task.BOTTOM_UP)
else:
model_cfg["method"] = str(Task.TOP_DOWN)
model_cfg["detector"] = config_utils.read_config_as_dict(
get_super_animal_model_config_path(detector_name)
)

# use SuperAnimal bodyparts
if weight_init.memory_replay:
Expand Down
6 changes: 3 additions & 3 deletions deeplabcut/pose_estimation_pytorch/modelzoo/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,7 @@ def update_config(config, max_individuals, device):
return config


def _parse_snapshot(snapshot: Path, device: str, print_keys: bool = False) -> None:
def _parse_snapshot(snapshot_path: Path, device: str, print_keys: bool = False) -> None:
"""FIXME: A new snapshot should be uploaded and used"""
def _map_model_keys(state_dict: dict) -> dict:
updated_dict = {}
Expand All @@ -218,7 +218,7 @@ def _map_model_keys(state_dict: dict) -> dict:
updated_dict[".".join(parts)] = v
return updated_dict

snapshot = torch.load(snapshot, map_location=device)
snapshot = torch.load(snapshot_path, map_location=device)
if print_keys:
print(5 * "-----\n")
print(snapshot.stem + " keys")
Expand All @@ -231,5 +231,5 @@ def _map_model_keys(state_dict: dict) -> dict:
model=_map_model_keys(snapshot["model_state_dict"]),
metadata=dict(epoch=0),
),
snapshot,
snapshot_path,
)