Visitar URL original
merge named snapshot evaluation into pytorch_dlc by n-poulsen · Pull Request #2657 · DeepLabCut/DeepLabCut · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Add feature to evaluate network for specfic, named snapshots. (#2508)
* Add snapshot selection option to evaluate_network while preserving existing functionality.
- Replace snapindex with more explicit snapshot_name. snapshotindex from config is used to identify the snapshot name instead.
- Refactor logic for snapshotindex allowed values. Add limit to negative index values.

* Update evaluate.py
- Replace snapshots_to_evaluate numpy indexing with list loop
- Raise error if no snapshots are found

* Add snapshots_to_evaluate docstring

* Remove unused distance function

* Update evaluate.py
* Improve snapshots_to_evaluate docstring
* Get training_iterations from snapshot_name
* Rename trainingsiterations -> training_iterations

* Improve snapshot_to_evaluate typehint and docstring.

* Refactor evaluate.py
* Create get_available_requested_snapshots for obtaining specified snapshot names (new feature).
* Create get_snapshots_by_index to handle the original implementation.
* Minor improvements and fixes

* Add tests for new snapshot functions in evaluate.py

* Update evaluate_multianimal.py
* Add snapshots_to_evaluate parameter to multi-animal evaluation
* Minor renaming

* Update python-package.yml

* add evaluate_by_snapshot branch for testing purposes

* Add snapshots_to_evaluate usage in testscript.py

* Update get_snapshots_by_index function
* Better typehint for int or str
* Raise IndexError instead of ValueError

* Add get_snapshots_by_index tests to test_evaluate.py

* Update return_evaluate_network_data function with get_snapshots_by_index usage

* Fix potential reference before assignment by raising ValueError

* Refactor getting existing snapshot file names as sorted list.
* Add list_sorted_existing_snapshots to auxiliaryfunctions.py.
* Update several files to use the newly refactored function.
* Preserve same exception handling, albeit with modified message to be more general and concise.

* Refactor evaluate_multianimal.py
* Add new snapshot auxiliary function usage.
* Preserve continuation of loop if no snapshots are found.

* Remove indentation level.

* Rename list_sorted_exisiting_snapshots -> get_snapshots_in_folder
* Update name for all usages
* Minor change to function docstring
* Remove redundant sorting in evaluate_multianimal.py

* Port numpy array to list (not performance critical)

* Add get_snapshots_from_folder tests
* Test snapshots present and returned in order
* Test snapshots not found

* Fix deprecated function names

* Improve snapshots_to_evaluate test in testscript.py

* Correct spelling

* Add per_keypoint_evaluation to evaluate_multianimal_full function call (woops!)

* minor linting

---------

Co-authored-by: Mackenzie Mathis <mathis@rowland.harvard.edu>
Co-authored-by: Niels Poulsen <niels.poulsen@epfl.ch>
  • Loading branch information
3 people authored Jul 5, 2024
commit 6277d77582cfe924c82f5d1eb33d841f5f09fa05
176 changes: 94 additions & 82 deletions deeplabcut/pose_estimation_tensorflow/core/evaluate.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
import argparse
import os
from pathlib import Path
from typing import List
from typing import List, Union
import numpy as np
import pandas as pd
from tqdm import tqdm
Expand Down Expand Up @@ -42,10 +42,6 @@ def pairwisedistances(DataCombined, scorer1, scorer2, pcutoff=-1, bodyparts=None
return RMSE, RMSE[mask]


def distance(v, w):
return np.sqrt(np.sum((v - w) ** 2))


def calculatepafdistancebounds(
config, shuffle=0, trainingsetindex=0, modelprefix="", numdigits=0, onlytrain=False
):
Expand Down Expand Up @@ -349,44 +345,25 @@ def return_evaluate_network_data(
)
),
)
# Check which snapshots are available and sort them by # iterations
Snapshots = np.array(
[
fn.split(".")[0]
for fn in os.listdir(os.path.join(str(modelfolder), "train"))
if "index" in fn
]

Snapshots = auxiliaryfunctions.get_snapshots_from_folder(
train_folder=Path(modelfolder) / "train",
)

if len(Snapshots) == 0:
print(
"Snapshots not found! It seems the dataset for shuffle %s and trainFraction %s is not trained.\nPlease train it before evaluating.\nUse the function 'train_network' to do so."
% (shuffle, trainFraction)
)
snapindices = []
else:
increasing_indices = np.argsort([int(m.split("-")[1]) for m in Snapshots])
Snapshots = Snapshots[increasing_indices]
if Snapindex is None:
Snapindex = cfg["snapshotindex"]

if Snapindex == -1:
snapindices = [-1]
elif Snapindex == "all":
snapindices = range(len(Snapshots))
elif Snapindex < len(Snapshots):
snapindices = [Snapindex]
else:
print(
"Invalid choice, only -1 (last), any integer up to last, or all (as string)!"
)
if Snapindex is None:
Snapindex = cfg["snapshotindex"]

snapshot_names = get_snapshots_by_index(
idx=Snapindex,
available_snapshots=Snapshots,
)

DATA = []
results = []
resultsfns = []
for snapindex in snapindices:
for snapshot_name in snapshot_names:
test_pose_cfg["init_weights"] = os.path.join(
str(modelfolder), "train", Snapshots[snapindex]
str(modelfolder), "train", snapshot_name
) # setting weights to corresponding snapshot.
trainingsiterations = (test_pose_cfg["init_weights"].split(os.sep)[-1]).split(
"-"
Expand All @@ -411,7 +388,7 @@ def return_evaluate_network_data(
resultsfilename,
DLCscorer,
) = auxiliaryfunctions.check_if_not_evaluated(
str(evaluationfolder), DLCscorer, DLCscorerlegacy, Snapshots[snapindex]
str(evaluationfolder), DLCscorer, DLCscorerlegacy, snapshot_name
)
# resultsfilename=os.path.join(str(evaluationfolder),DLCscorer + '-' + str(Snapshots[snapindex])+ '.h5') # + '-' + str(snapshot)+ ' #'-' + Snapshots[snapindex]+ '.h5')
print(resultsfilename)
Expand Down Expand Up @@ -458,7 +435,7 @@ def return_evaluate_network_data(
np.round(testerrorpcutoff, 2),
"pixels",
)
print("Snapshot", Snapshots[snapindex])
print("Snapshot", snapshot_name)

r = [
trainingsiterations,
Expand All @@ -469,7 +446,7 @@ def return_evaluate_network_data(
cfg["pcutoff"],
np.round(trainerrorpcutoff, 2),
np.round(testerrorpcutoff, 2),
Snapshots[snapindex],
snapshot_name,
scale,
test_pose_cfg["net_type"],
]
Expand All @@ -489,7 +466,7 @@ def return_evaluate_network_data(
comparisonbodyparts,
cfg,
evaluationfolder,
Snapshots[snapindex],
snapshot_name,
]
)

Expand Down Expand Up @@ -563,6 +540,7 @@ def evaluate_network(
rescale=False,
modelprefix="",
per_keypoint_evaluation: bool = False,
snapshots_to_evaluate: List[str] = None,
):
"""Evaluates the network.

Expand Down Expand Up @@ -620,6 +598,9 @@ def evaluate_network(
Compute the train and test RMSE for each keypoint, and save the results to
a {model_name}-keypoint-results.csv in the evalution-results folder

snapshots_to_evaluate: List[str], optional, default=None
List of snapshot names to evaluate (e.g. ["snapshot-50000", "snapshot-75000", ...])

Returns
-------
None
Expand Down Expand Up @@ -673,6 +654,7 @@ def evaluate_network(
gputouse=gputouse,
modelprefix=modelprefix,
per_keypoint_evaluation=per_keypoint_evaluation,
snapshots_to_evaluate=snapshots_to_evaluate,
)
else:
from deeplabcut.utils.auxfun_videos import imread, imresize
Expand Down Expand Up @@ -787,36 +769,19 @@ def evaluate_network(
evaluationfolder, recursive=True
)

# Check which snapshots are available and sort them by # iterations
Snapshots = np.array(
[
fn.split(".")[0]
for fn in os.listdir(os.path.join(str(modelfolder), "train"))
if "index" in fn
]
Snapshots = auxiliaryfunctions.get_snapshots_from_folder(
train_folder=Path(modelfolder) / "train",
)
try: # check if any where found?
Snapshots[0]
except IndexError:
raise FileNotFoundError(
"Snapshots not found! It seems the dataset for shuffle %s and trainFraction %s is not trained.\nPlease train it before evaluating.\nUse the function 'train_network' to do so."
% (shuffle, trainFraction)
)

increasing_indices = np.argsort(
[int(m.split("-")[1]) for m in Snapshots]
)
Snapshots = Snapshots[increasing_indices]

if cfg["snapshotindex"] == -1:
snapindices = [-1]
elif cfg["snapshotindex"] == "all":
snapindices = range(len(Snapshots))
elif cfg["snapshotindex"] < len(Snapshots):
snapindices = [cfg["snapshotindex"]]
if snapshots_to_evaluate is not None:
snapshot_names = get_available_requested_snapshots(
requested_snapshots=snapshots_to_evaluate,
available_snapshots=Snapshots,
)
else:
raise ValueError(
"Invalid choice, only -1 (last), any integer up to last, or all (as string)!"
snapshot_names = get_snapshots_by_index(
idx=cfg["snapshotindex"],
available_snapshots=Snapshots,
)

final_result = []
Expand All @@ -841,29 +806,25 @@ def evaluate_network(
##################################################
# Compute predictions over images
##################################################
for snapindex in snapindices:
for snapshot_name in snapshot_names:
test_pose_cfg["init_weights"] = os.path.join(
str(modelfolder), "train", Snapshots[snapindex]
str(modelfolder), "train", snapshot_name
) # setting weights to corresponding snapshot.
trainingsiterations = (
test_pose_cfg["init_weights"].split(os.sep)[-1]
).split("-")[
-1
] # read how many training siterations that corresponds to.
training_iterations = int(snapshot_name.split("-")[-1])

# Name for deeplabcut net (based on its parameters)
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.get_scorer_name(
cfg,
shuffle,
trainFraction,
trainingsiterations,
training_iterations,
modelprefix=modelprefix,
)
print(
"Running ",
DLCscorer,
" with # of training iterations:",
trainingsiterations,
training_iterations,
)
(
notanalyzed,
Expand All @@ -873,7 +834,7 @@ def evaluate_network(
str(evaluationfolder),
DLCscorer,
DLCscorerlegacy,
Snapshots[snapindex],
snapshot_name,
)
if notanalyzed:
# Specifying state of model (snapshot / training state)
Expand Down Expand Up @@ -931,7 +892,7 @@ def evaluate_network(

print(
"Analysis is done and the results are stored (see evaluation-results) for snapshot: ",
Snapshots[snapindex],
snapshot_name,
)
DataCombined = pd.concat(
[Data.T, DataMachine.T], axis=0, sort=False
Expand All @@ -955,7 +916,7 @@ def evaluate_network(
RMSEpcutoff.iloc[trainIndices].values.flatten()
)
results = [
trainingsiterations,
training_iterations,
int(100 * trainFraction),
shuffle,
np.round(trainerror, 2),
Expand All @@ -978,7 +939,7 @@ def evaluate_network(
if show_errors:
print(
"Results for",
trainingsiterations,
training_iterations,
" training iterations:",
int(100 * trainFraction),
shuffle,
Expand Down Expand Up @@ -1013,7 +974,7 @@ def evaluate_network(
"LabeledImages_"
+ DLCscorer
+ "_"
+ Snapshots[snapindex],
+ snapshot_name,
)
auxiliaryfunctions.attempt_to_make_folder(foldername)
Plotting(
Expand All @@ -1039,7 +1000,7 @@ def evaluate_network(
"LabeledImages_"
+ DLCscorer
+ "_"
+ Snapshots[snapindex],
+ snapshot_name,
)
if not os.path.exists(foldername):
print(
Expand Down Expand Up @@ -1112,6 +1073,57 @@ def make_results_file(final_result, evaluationfolder, DLCscorer):
df.to_csv(output_path)


def get_available_requested_snapshots(
requested_snapshots: List[str],
available_snapshots: List[str],
) -> List[str]:
"""
Intersects the requested snapshot names with the available snapshots.

Returns: snapshot names
"""
snapshot_names = []
missing_snapshots = []
for snap in requested_snapshots:
if snap in available_snapshots:
snapshot_names.append(snap)
else:
missing_snapshots.append(snap)

if len(snapshot_names) == 0:
raise ValueError(
f"None of the requested snapshots were found: \n{missing_snapshots}"
)
elif len(missing_snapshots) > 0:
print(
f"The following requested snapshots were not found and will be skipped:\n"
f"{missing_snapshots}"
)

return snapshot_names


def get_snapshots_by_index(
idx: Union[int, str], available_snapshots: List[str],
) -> List[str]:
"""
Assume available_snapshots is ordered in ascending order. Returns snapshot names.
"""
if (
isinstance(idx, int)
and -len(available_snapshots) <= idx < len(available_snapshots)
):
return [available_snapshots[idx]]
elif idx == "all":
return available_snapshots

raise IndexError(
f"Invalid index: {idx}. The index should be an int less than the number of "
f"available snapshots, negative indexing is supported. The keyword 'all' "
f"is also a valid option."
)


if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("config")
Expand Down
Loading