Visitar URL original
merge named snapshot evaluation into pytorch_dlc (#2657) · DeepLabCut/DeepLabCut@61cfdba · GitHub
Skip to content

Commit 61cfdba

Browse files
n-poulsenTetra-quarkMMathisLabpark-jsdev
authored
merge named snapshot evaluation into pytorch_dlc (#2657)
* Add feature to evaluate network for specfic, named snapshots. (#2508) * Add snapshot selection option to evaluate_network while preserving existing functionality. - Replace snapindex with more explicit snapshot_name. snapshotindex from config is used to identify the snapshot name instead. - Refactor logic for snapshotindex allowed values. Add limit to negative index values. * Update evaluate.py - Replace snapshots_to_evaluate numpy indexing with list loop - Raise error if no snapshots are found * Add snapshots_to_evaluate docstring * Remove unused distance function * Update evaluate.py * Improve snapshots_to_evaluate docstring * Get training_iterations from snapshot_name * Rename trainingsiterations -> training_iterations * Improve snapshot_to_evaluate typehint and docstring. * Refactor evaluate.py * Create get_available_requested_snapshots for obtaining specified snapshot names (new feature). * Create get_snapshots_by_index to handle the original implementation. * Minor improvements and fixes * Add tests for new snapshot functions in evaluate.py * Update evaluate_multianimal.py * Add snapshots_to_evaluate parameter to multi-animal evaluation * Minor renaming * Update python-package.yml * add evaluate_by_snapshot branch for testing purposes * Add snapshots_to_evaluate usage in testscript.py * Update get_snapshots_by_index function * Better typehint for int or str * Raise IndexError instead of ValueError * Add get_snapshots_by_index tests to test_evaluate.py * Update return_evaluate_network_data function with get_snapshots_by_index usage * Fix potential reference before assignment by raising ValueError * Refactor getting existing snapshot file names as sorted list. * Add list_sorted_existing_snapshots to auxiliaryfunctions.py. * Update several files to use the newly refactored function. * Preserve same exception handling, albeit with modified message to be more general and concise. * Refactor evaluate_multianimal.py * Add new snapshot auxiliary function usage. * Preserve continuation of loop if no snapshots are found. * Remove indentation level. * Rename list_sorted_exisiting_snapshots -> get_snapshots_in_folder * Update name for all usages * Minor change to function docstring * Remove redundant sorting in evaluate_multianimal.py * Port numpy array to list (not performance critical) * Add get_snapshots_from_folder tests * Test snapshots present and returned in order * Test snapshots not found * Fix deprecated function names * Improve snapshots_to_evaluate test in testscript.py * Correct spelling * Add per_keypoint_evaluation to evaluate_multianimal_full function call (woops!) * minor linting --------- Co-authored-by: Mackenzie Mathis <mathis@rowland.harvard.edu> Co-authored-by: Niels Poulsen <niels.poulsen@epfl.ch> * fixed file dialog not showing uppercase file extension video files (#2645) * fixed file dialog not showing uppercase file extension video files * bug fix: no shuffle_change slot in main window * bug fix --------- Co-authored-by: Niels Poulsen <niels.poulsen@epfl.ch> Co-authored-by: Mackenzie Mathis <mathis@rowland.harvard.edu> --------- Co-authored-by: jkopp <32641315+Tetra-quark@users.noreply.github.com> Co-authored-by: Mackenzie Mathis <mathis@rowland.harvard.edu> Co-authored-by: Terry Park <58895710+park-jsdev@users.noreply.github.com>
1 parent 20dac7d commit 61cfdba

11 files changed

Lines changed: 687 additions & 608 deletions

File tree

‎deeplabcut/compat.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -327,6 +327,7 @@ def evaluate_network(
327327
rescale: bool = False,
328328
modelprefix: str = "",
329329
per_keypoint_evaluation: bool = False,
330+
snapshots_to_evaluate: list[str] | None = None,
330331
engine: Engine | None = None,
331332
**torch_kwargs,
332333
):
@@ -386,6 +387,9 @@ def evaluate_network(
386387
Compute the train and test RMSE for each keypoint, and save the results to
387388
a {model_name}-keypoint-results.csv in the evalution-results folder
388389
390+
snapshots_to_evaluate: List[str], optional, default=None
391+
List of snapshot names to evaluate (e.g. ["snapshot-5000", "snapshot-7500"]).
392+
389393
engine: Engine, optional, default = None.
390394
The default behavior loads the engine for the shuffle from the metadata. You can
391395
overwrite this by passing the engine as an argument, but this should generally
@@ -464,6 +468,7 @@ def evaluate_network(
464468
rescale=rescale,
465469
modelprefix=modelprefix,
466470
per_keypoint_evaluation=per_keypoint_evaluation,
471+
snapshots_to_evaluate=snapshots_to_evaluate,
467472
)
468473
elif engine == Engine.PYTORCH:
469474
from deeplabcut.pose_estimation_pytorch.apis import evaluate_network

‎deeplabcut/gui/components.py‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -160,11 +160,19 @@ def _update_video_selection(self, videopaths):
160160

161161
def select_videos(self):
162162
cwd = self.root.project_folder
163+
164+
# Create a filter string with both lowercase and uppercase extensions
165+
166+
video_types = [f"*.{ext.lower()}" for ext in DLCParams.VIDEOTYPES[1:]] + [
167+
f"*.{ext.upper()}" for ext in DLCParams.VIDEOTYPES[1:]
168+
]
169+
video_files = f"Videos ({' '.join(video_types)})"
170+
163171
filenames = QtWidgets.QFileDialog.getOpenFileNames(
164172
self,
165173
"Select video(s) to analyze",
166174
cwd,
167-
f"Videos ({' *.'.join(DLCParams.VIDEOTYPES)[1:]})",
175+
video_files,
168176
)
169177

170178
if filenames[0]:

‎deeplabcut/pose_estimation_tensorflow/core/evaluate.py‎

Lines changed: 95 additions & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,8 @@
1313
import argparse
1414
import os
1515
from pathlib import Path
16-
from typing import List
16+
from typing import List, Union
17+
1718
import numpy as np
1819
import pandas as pd
1920
from tqdm import tqdm
@@ -42,10 +43,6 @@ def pairwisedistances(DataCombined, scorer1, scorer2, pcutoff=-1, bodyparts=None
4243
return RMSE, RMSE[mask]
4344

4445

45-
def distance(v, w):
46-
return np.sqrt(np.sum((v - w) ** 2))
47-
48-
4946
def calculatepafdistancebounds(
5047
config, shuffle=0, trainingsetindex=0, modelprefix="", numdigits=0, onlytrain=False
5148
):
@@ -349,44 +346,25 @@ def return_evaluate_network_data(
349346
)
350347
),
351348
)
352-
# Check which snapshots are available and sort them by # iterations
353-
Snapshots = np.array(
354-
[
355-
fn.split(".")[0]
356-
for fn in os.listdir(os.path.join(str(modelfolder), "train"))
357-
if "index" in fn
358-
]
349+
350+
Snapshots = auxiliaryfunctions.get_snapshots_from_folder(
351+
train_folder=Path(modelfolder) / "train",
359352
)
360353

361-
if len(Snapshots) == 0:
362-
print(
363-
"Snapshots not found! It seems the dataset for shuffle %s and trainFraction %s is not trained.\nPlease train it before evaluating.\nUse the function 'train_network' to do so."
364-
% (shuffle, trainFraction)
365-
)
366-
snapindices = []
367-
else:
368-
increasing_indices = np.argsort([int(m.split("-")[1]) for m in Snapshots])
369-
Snapshots = Snapshots[increasing_indices]
370-
if Snapindex is None:
371-
Snapindex = cfg["snapshotindex"]
372-
373-
if Snapindex == -1:
374-
snapindices = [-1]
375-
elif Snapindex == "all":
376-
snapindices = range(len(Snapshots))
377-
elif Snapindex < len(Snapshots):
378-
snapindices = [Snapindex]
379-
else:
380-
print(
381-
"Invalid choice, only -1 (last), any integer up to last, or all (as string)!"
382-
)
354+
if Snapindex is None:
355+
Snapindex = cfg["snapshotindex"]
356+
357+
snapshot_names = get_snapshots_by_index(
358+
idx=Snapindex,
359+
available_snapshots=Snapshots,
360+
)
383361

384362
DATA = []
385363
results = []
386364
resultsfns = []
387-
for snapindex in snapindices:
365+
for snapshot_name in snapshot_names:
388366
test_pose_cfg["init_weights"] = os.path.join(
389-
str(modelfolder), "train", Snapshots[snapindex]
367+
str(modelfolder), "train", snapshot_name
390368
) # setting weights to corresponding snapshot.
391369
trainingsiterations = (test_pose_cfg["init_weights"].split(os.sep)[-1]).split(
392370
"-"
@@ -411,7 +389,7 @@ def return_evaluate_network_data(
411389
resultsfilename,
412390
DLCscorer,
413391
) = auxiliaryfunctions.check_if_not_evaluated(
414-
str(evaluationfolder), DLCscorer, DLCscorerlegacy, Snapshots[snapindex]
392+
str(evaluationfolder), DLCscorer, DLCscorerlegacy, snapshot_name
415393
)
416394
# resultsfilename=os.path.join(str(evaluationfolder),DLCscorer + '-' + str(Snapshots[snapindex])+ '.h5') # + '-' + str(snapshot)+ ' #'-' + Snapshots[snapindex]+ '.h5')
417395
print(resultsfilename)
@@ -458,7 +436,7 @@ def return_evaluate_network_data(
458436
np.round(testerrorpcutoff, 2),
459437
"pixels",
460438
)
461-
print("Snapshot", Snapshots[snapindex])
439+
print("Snapshot", snapshot_name)
462440

463441
r = [
464442
trainingsiterations,
@@ -469,7 +447,7 @@ def return_evaluate_network_data(
469447
cfg["pcutoff"],
470448
np.round(trainerrorpcutoff, 2),
471449
np.round(testerrorpcutoff, 2),
472-
Snapshots[snapindex],
450+
snapshot_name,
473451
scale,
474452
test_pose_cfg["net_type"],
475453
]
@@ -489,7 +467,7 @@ def return_evaluate_network_data(
489467
comparisonbodyparts,
490468
cfg,
491469
evaluationfolder,
492-
Snapshots[snapindex],
470+
snapshot_name,
493471
]
494472
)
495473

@@ -563,6 +541,7 @@ def evaluate_network(
563541
rescale=False,
564542
modelprefix="",
565543
per_keypoint_evaluation: bool = False,
544+
snapshots_to_evaluate: List[str] = None,
566545
):
567546
"""Evaluates the network.
568547
@@ -620,6 +599,9 @@ def evaluate_network(
620599
Compute the train and test RMSE for each keypoint, and save the results to
621600
a {model_name}-keypoint-results.csv in the evalution-results folder
622601
602+
snapshots_to_evaluate: List[str], optional, default=None
603+
List of snapshot names to evaluate (e.g. ["snapshot-50000", "snapshot-75000", ...])
604+
623605
Returns
624606
-------
625607
None
@@ -673,6 +655,7 @@ def evaluate_network(
673655
gputouse=gputouse,
674656
modelprefix=modelprefix,
675657
per_keypoint_evaluation=per_keypoint_evaluation,
658+
snapshots_to_evaluate=snapshots_to_evaluate,
676659
)
677660
else:
678661
from deeplabcut.utils.auxfun_videos import imread, imresize
@@ -787,36 +770,19 @@ def evaluate_network(
787770
evaluationfolder, recursive=True
788771
)
789772

790-
# Check which snapshots are available and sort them by # iterations
791-
Snapshots = np.array(
792-
[
793-
fn.split(".")[0]
794-
for fn in os.listdir(os.path.join(str(modelfolder), "train"))
795-
if "index" in fn
796-
]
773+
Snapshots = auxiliaryfunctions.get_snapshots_from_folder(
774+
train_folder=Path(modelfolder) / "train",
797775
)
798-
try: # check if any where found?
799-
Snapshots[0]
800-
except IndexError:
801-
raise FileNotFoundError(
802-
"Snapshots not found! It seems the dataset for shuffle %s and trainFraction %s is not trained.\nPlease train it before evaluating.\nUse the function 'train_network' to do so."
803-
% (shuffle, trainFraction)
804-
)
805776

806-
increasing_indices = np.argsort(
807-
[int(m.split("-")[1]) for m in Snapshots]
808-
)
809-
Snapshots = Snapshots[increasing_indices]
810-
811-
if cfg["snapshotindex"] == -1:
812-
snapindices = [-1]
813-
elif cfg["snapshotindex"] == "all":
814-
snapindices = range(len(Snapshots))
815-
elif cfg["snapshotindex"] < len(Snapshots):
816-
snapindices = [cfg["snapshotindex"]]
777+
if snapshots_to_evaluate is not None:
778+
snapshot_names = get_available_requested_snapshots(
779+
requested_snapshots=snapshots_to_evaluate,
780+
available_snapshots=Snapshots,
781+
)
817782
else:
818-
raise ValueError(
819-
"Invalid choice, only -1 (last), any integer up to last, or all (as string)!"
783+
snapshot_names = get_snapshots_by_index(
784+
idx=cfg["snapshotindex"],
785+
available_snapshots=Snapshots,
820786
)
821787

822788
final_result = []
@@ -841,29 +807,25 @@ def evaluate_network(
841807
##################################################
842808
# Compute predictions over images
843809
##################################################
844-
for snapindex in snapindices:
810+
for snapshot_name in snapshot_names:
845811
test_pose_cfg["init_weights"] = os.path.join(
846-
str(modelfolder), "train", Snapshots[snapindex]
812+
str(modelfolder), "train", snapshot_name
847813
) # setting weights to corresponding snapshot.
848-
trainingsiterations = (
849-
test_pose_cfg["init_weights"].split(os.sep)[-1]
850-
).split("-")[
851-
-1
852-
] # read how many training siterations that corresponds to.
814+
training_iterations = int(snapshot_name.split("-")[-1])
853815

854816
# Name for deeplabcut net (based on its parameters)
855817
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.get_scorer_name(
856818
cfg,
857819
shuffle,
858820
trainFraction,
859-
trainingsiterations=trainingsiterations,
821+
trainingsiterations=training_iterations,
860822
modelprefix=modelprefix,
861823
)
862824
print(
863825
"Running ",
864826
DLCscorer,
865827
" with # of training iterations:",
866-
trainingsiterations,
828+
training_iterations,
867829
)
868830
(
869831
notanalyzed,
@@ -873,7 +835,7 @@ def evaluate_network(
873835
str(evaluationfolder),
874836
DLCscorer,
875837
DLCscorerlegacy,
876-
Snapshots[snapindex],
838+
snapshot_name,
877839
)
878840
if notanalyzed:
879841
# Specifying state of model (snapshot / training state)
@@ -931,7 +893,7 @@ def evaluate_network(
931893

932894
print(
933895
"Analysis is done and the results are stored (see evaluation-results) for snapshot: ",
934-
Snapshots[snapindex],
896+
snapshot_name,
935897
)
936898
DataCombined = pd.concat(
937899
[Data.T, DataMachine.T], axis=0, sort=False
@@ -955,7 +917,7 @@ def evaluate_network(
955917
RMSEpcutoff.iloc[trainIndices].values.flatten()
956918
)
957919
results = [
958-
trainingsiterations,
920+
training_iterations,
959921
int(100 * trainFraction),
960922
shuffle,
961923
np.round(trainerror, 2),
@@ -978,7 +940,7 @@ def evaluate_network(
978940
if show_errors:
979941
print(
980942
"Results for",
981-
trainingsiterations,
943+
training_iterations,
982944
" training iterations:",
983945
int(100 * trainFraction),
984946
shuffle,
@@ -1013,7 +975,7 @@ def evaluate_network(
1013975
"LabeledImages_"
1014976
+ DLCscorer
1015977
+ "_"
1016-
+ Snapshots[snapindex],
978+
+ snapshot_name,
1017979
)
1018980
auxiliaryfunctions.attempt_to_make_folder(foldername)
1019981
Plotting(
@@ -1039,7 +1001,7 @@ def evaluate_network(
10391001
"LabeledImages_"
10401002
+ DLCscorer
10411003
+ "_"
1042-
+ Snapshots[snapindex],
1004+
+ snapshot_name,
10431005
)
10441006
if not os.path.exists(foldername):
10451007
print(
@@ -1112,6 +1074,57 @@ def make_results_file(final_result, evaluationfolder, DLCscorer):
11121074
df.to_csv(output_path)
11131075

11141076

1077+
def get_available_requested_snapshots(
1078+
requested_snapshots: List[str],
1079+
available_snapshots: List[str],
1080+
) -> List[str]:
1081+
"""
1082+
Intersects the requested snapshot names with the available snapshots.
1083+
1084+
Returns: snapshot names
1085+
"""
1086+
snapshot_names = []
1087+
missing_snapshots = []
1088+
for snap in requested_snapshots:
1089+
if snap in available_snapshots:
1090+
snapshot_names.append(snap)
1091+
else:
1092+
missing_snapshots.append(snap)
1093+
1094+
if len(snapshot_names) == 0:
1095+
raise ValueError(
1096+
f"None of the requested snapshots were found: \n{missing_snapshots}"
1097+
)
1098+
elif len(missing_snapshots) > 0:
1099+
print(
1100+
f"The following requested snapshots were not found and will be skipped:\n"
1101+
f"{missing_snapshots}"
1102+
)
1103+
1104+
return snapshot_names
1105+
1106+
1107+
def get_snapshots_by_index(
1108+
idx: Union[int, str], available_snapshots: List[str],
1109+
) -> List[str]:
1110+
"""
1111+
Assume available_snapshots is ordered in ascending order. Returns snapshot names.
1112+
"""
1113+
if (
1114+
isinstance(idx, int)
1115+
and -len(available_snapshots) <= idx < len(available_snapshots)
1116+
):
1117+
return [available_snapshots[idx]]
1118+
elif idx == "all":
1119+
return available_snapshots
1120+
1121+
raise IndexError(
1122+
f"Invalid index: {idx}. The index should be an int less than the number of "
1123+
f"available snapshots, negative indexing is supported. The keyword 'all' "
1124+
f"is also a valid option."
1125+
)
1126+
1127+
11151128
if __name__ == "__main__":
11161129
parser = argparse.ArgumentParser()
11171130
parser.add_argument("config")

0 commit comments

Comments
 (0)