Visitar URL original
New PyTorch DeepLabCut SuperAnimal demo notebook by yeshaokai · Pull Request #2629 · DeepLabCut/DeepLabCut · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
70a0155
spatial pyramid for video adaptation by default, with a small range o…
yeshaokai Apr 6, 2023
8441909
Update test_crossvalutils.py
yeshaokai Apr 20, 2023
0e20e8d
Merge branch 'DeepLabCut:main' into master
yeshaokai Apr 25, 2023
d5febb1
Handled empty prediction
yeshaokai Apr 25, 2023
f32d317
Removed unused parameter. Updated config to use superanimal instead o…
May 23, 2023
7949045
Merge branch 'DeepLabCut:main' into master
yeshaokai May 31, 2023
d531e80
Merge branch 'DeepLabCut:main' into master
yeshaokai Jul 30, 2023
768296c
Merge branch 'DeepLabCut:main' into master
yeshaokai Jun 20, 2024
882f920
Added notebook for superanimal pytorch
yeshaokai Jun 20, 2024
4be4a70
Update COLAB_Pytorch_SuperAnimal.ipynb
MMathisLab Jun 20, 2024
ed8a979
Update COLAB_Pytorch_SuperAnimal.ipynb
MMathisLab Jun 20, 2024
624134b
Update COLAB_Pytorch_SuperAnimal.ipynb
MMathisLab Jun 20, 2024
7c123b3
Merge branch 'pytorch_dlc' into shaokai/educational_superanimal_notebook
Jun 26, 2024
d68a137
cleaned notebook and added bbox_threshold to analyze_superanimal_images
Jun 26, 2024
830f4ac
worked on superanimal notebook
Jul 2, 2024
66acb3d
fixed symlink creation for keypoint matching
Jul 4, 2024
ba39fd5
renamed COLAB to YOURDATA
Jul 4, 2024
24cc195
fixed some bugs and cleaned the notebook. added more print statements…
Jul 4, 2024
c99a199
installed latest version of DeepLabCut instead of from pypi
Jul 4, 2024
f59dc69
changed default training parameters
Jul 4, 2024
3877be3
added option to not have use symlinks for keypoint matching
Jul 4, 2024
97bb2a1
no symlinks for keypoint matching in COLAB
Jul 4, 2024
67903f0
set default eval interval to 10, add superanimal_analyze_images to init
Jul 4, 2024
faba3ac
do not copy images or create symlinks for memory replay
Jul 4, 2024
27a58af
fixed memory replay to only run once
Jul 9, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fixed symlink creation for keypoint matching
  • Loading branch information
Niels Poulsen
Niels Poulsen committed Jul 4, 2024
commit 66acb3d3d4baa2814c4a329adde817e0155184f0
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import os
import pickle
import shutil
from pathlib import Path

import numpy as np
import pandas as pd
Expand Down Expand Up @@ -693,52 +694,42 @@ def _generic2coco(
broken_links = []
# copying images via symbolic link
for image in train_images + test_images:
src = image["file_name"]
# important to resolve the filepath! Otherwise, errors can occur when running
# this code from Jupyter Notebooks
src = Path(image["file_name"]).resolve()
image_id = image["id"]

if not os.path.exists(src):
if not src.exists():
print("problem comes from", image["source_dataset"])
print(src)
broken_links.append(image_id)
continue
else:
pass
# print ('success comes from', image['source_dataset'])
# print (src)

# in dlc, some images have same name but under different folder
# we used to use a parent folder to distinguish them, but it's only applicable to DLC
# so here it's easier to just append a id into the filename

image_name = src.split(os.sep)[-1]

if image_name.count(".") > 1:
sep = image_name.rfind(".")
pre, suffix = image_name[:sep], image_name[sep + 1 :]
else:
# this does not work for image file that looks like image9.5.jpg..
pre, suffix = image_name.split(".")
# we used to use a parent folder to distinguish them, but it's only applicable
# to DLC so here it's easier to just append a id into the filename

# not to repeatedly add image id in memory replay training
dest_image_name = src.name
if append_image_id:
dest_image_name = f"{pre}_{image_id}.{suffix}"
else:
dest_image_name = image_name
dest = os.path.join(proj_root, "images", dest_image_name)
dest_image_name = f"{src.stem}_{image_id}{src.suffix}"

# now, we will also need to update the path in the config files
dest = Path(proj_root) / "images" / dest_image_name
dest = dest.resolve()

# now, we will also need to update the path in the config files
if full_image_path:
image["file_name"] = dest
image["file_name"] = str(dest)
else:
image["file_name"] = os.path.join("images", dest_image_name)
image["file_name"] = str(Path(*dest.parts[-2:]))

if deepcopy:
shutil.copy(src, dest)
else:
try:
os.symlink(src, dest)
except:
except Exception as err:
print(f"Could not create a symlink from {src} to {dest}: {err}")
pass

lookuptable[dest] = src
Expand Down
93 changes: 46 additions & 47 deletions deeplabcut/utils/pseudo_label.py
Original file line number Diff line number Diff line change
Expand Up @@ -145,29 +145,36 @@ def plot_cost_matrix(


def keypoint_matching(
config_path,
superanimal_name,
model_name,
device=None,
train_file="train.json",
pose_threshold=0.1,
config_path: str | Path,
superanimal_name: str,
model_name: str,
device: str | None = None,
train_file: str = "train.json",
):
"""Runs the keypoint matching algorithm for a DeepLabCut project

cfg = af.read_config(config_path)

trainIndex = 0
Matches project keypoints to SuperAnimal keypoints automatically, by running
SuperAnimal inference on all images in the dataset

dlc_proj_root = str(Path(config_path).parent)
Args:
config_path: the path of the DeepLabCut project configuration file
superanimal_name: SuperAnimal dataset with which to run keypoint matching
model_name: SuperAnimal model with which to run keypoint matching
device: the device on which to run keypoint matching
train_file: the name of the file containing the labels to output
"""
config_path = Path(config_path)
cfg = af.read_config(str(config_path))
dlc_proj_root = config_path.parent

if "individuals" in cfg:
temp_dataset = MaDLCDataFrame(dlc_proj_root, "temp_dataset")
temp_dataset = MaDLCDataFrame(str(dlc_proj_root), "temp_dataset")
max_individuals = len(cfg["individuals"])
else:
temp_dataset = SingleDLCDataFrame(dlc_proj_root, "temp_dataset")
temp_dataset = SingleDLCDataFrame(str(dlc_proj_root), "temp_dataset")
max_individuals = 1

memory_replay_folder = Path(dlc_proj_root) / "memory_replay"

memory_replay_folder = dlc_proj_root / "memory_replay"
temp_dataset.materialize(str(memory_replay_folder), framework="coco")

# inferencing the train set
Expand All @@ -186,7 +193,6 @@ def keypoint_matching(

individuals = [f"animal{i}" for i in range(max_individuals)]
config["individuals"] = individuals
num_bodyparts = len(config["bodyparts"])
train_file_path = os.path.join(memory_replay_folder, "annotations", train_file)

pose_runner, detector_runner = get_inference_runners(
Expand All @@ -204,30 +210,29 @@ def keypoint_matching(
images = train_obj["images"]
annotations = train_obj["annotations"]
categories = train_obj["categories"]
imagename2id = {}
imageid2name = {}
imagename2gt = defaultdict(list)
image_name_to_id = {}
image_id_to_name = {}

image_name_to_gt = defaultdict(list)
image_name_to_bbox = defaultdict(list)
image_id_to_annotations = defaultdict(list)

for image in images:
# this only works with relative path as the testing image can be at a different folder
imagename = image["file_name"].split(os.sep)[-1]
imagename2id[imagename] = image["id"]
imageid2name[image["id"]] = imagename

imagename2bbox = defaultdict(list)
name = image["file_name"].split(os.sep)[-1]
image_name_to_id[name] = image["id"]
image_id_to_name[image["id"]] = name

for anno in annotations:
imagename = imageid2name[anno["image_id"]]
imagename2gt[imagename].append(anno)
imagename2bbox[imagename].append(anno["bbox"])

imageid2annotations = defaultdict(list)
name = image_id_to_name[anno["image_id"]]
image_name_to_gt[name].append(anno)
image_name_to_bbox[name].append(anno["bbox"])

imageids = list(imagename2id.values())
for annotation in annotations:
image_id = annotation["image_id"]
if annotation["image_id"] in imageids:
imageid2annotations[image_id].append(annotation)
image_ids = set(image_name_to_id.values())
for anno in annotations:
image_id = anno["image_id"]
if anno["image_id"] in image_ids:
image_id_to_annotations[image_id].append(anno)

# need to support more image types
image_extensions = ["*.png", "*.jpg", "*.jpeg", "*.bmp", "*.gif", "*.tiff"]
Expand All @@ -238,19 +243,15 @@ def keypoint_matching(
)

corresponded_images = []

for image in images_in_folder:
image_path = image
imagename = image.split(os.sep)[-1]
if imagename in imagename2id:
name = image.split(os.sep)[-1]
if name in image_name_to_id:
corresponded_images.append(image_path)

images = corresponded_images

bbox_predictions = detector_runner.inference(images=images)

bbox_gts = [
{"bboxes": np.array(imagename2bbox[image.split(os.sep)[-1]])}
{"bboxes": np.array(image_name_to_bbox[image.split(os.sep)[-1]])}
for image in images
]

Expand All @@ -260,16 +261,14 @@ def keypoint_matching(
predictions = pose_runner.inference(pose_inputs)

with open(str(memory_replay_folder / "pseudo_predictions.json"), "w") as f:

json.dump(pose_inputs, f, cls=NumpyEncoder)

assert len(images) == len(predictions)

imagename2prediction = {}

image_name_to_pred = {}
for image_path, prediction in zip(images, predictions):
imagename = image_path.split(os.sep)[-1]
imagename2prediction[imagename] = prediction
name = image_path.split(os.sep)[-1]
image_name_to_pred[name] = prediction

pred_keypoint_names = config["bodyparts"]
num_pred_keypoints = len(pred_keypoint_names)
Expand All @@ -279,10 +278,10 @@ def keypoint_matching(
match_matrix = np.zeros((num_pred_keypoints, num_gt_keypoints))
match_dict = defaultdict(lambda: defaultdict(int))

for imagename, gts in imagename2gt.items():
for name, gts in image_name_to_gt.items():
bbox_gts = [np.array(gt["bbox"]) for gt in gts]
bbox_gts = [xywh2xyxy(e) for e in bbox_gts]
prediction = imagename2prediction[imagename]
prediction = image_name_to_pred[name]
bbox_preds = [xywh2xyxy(pred) for pred in prediction["bboxes"]]
optimal_pred_indices = optimal_match(bbox_gts, bbox_preds)

Expand Down