Visitar URL original
Data compatibility - Benchmark by n-poulsen · Pull Request #2740 · DeepLabCut/DeepLabCut · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion deeplabcut/benchmark/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,11 @@ def evaluate(
continue
benchmark = benchmark_cls()
for name in benchmark.names():
if Result(method_name=name, benchmark_name=benchmark_cls.name) in results:
if Result(
code=benchmark.code,
method_name=name,
benchmark_name=benchmark_cls.name,
) in results:
continue
else:
result = benchmark.evaluate(name, on_error=on_error)
Expand Down
5 changes: 4 additions & 1 deletion deeplabcut/benchmark/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ def get_predictions(self):
raise NotImplementedError()

def __init__(self):
keys = ["name", "keypoints", "ground_truth", "metadata"]
keys = ["code", "name", "keypoints", "ground_truth", "metadata"]
for key in keys:
if not hasattr(self, key):
raise NotImplementedError(
Expand Down Expand Up @@ -110,6 +110,7 @@ def evaluate(self, name: str, on_error="raise"):
else:
raise NotImplementedError() from exception
return Result(
code=self.code,
method_name=name,
benchmark_name=self.name,
mean_avg_precision=mean_avg_precision,
Expand Down Expand Up @@ -139,13 +140,15 @@ def _validate_predictions(self, name: str, predictions: dict) -> dict:
class Result:
"""Benchmark result."""

code: str
method_name: str
benchmark_name: str
root_mean_squared_error: float = float("nan")
mean_avg_precision: float = float("nan")
benchmark_version: str = __version__

_export_mapping = dict(
code="code",
benchmark_name="benchmark",
method_name="method",
benchmark_version="version",
Expand Down
10 changes: 10 additions & 0 deletions deeplabcut/benchmark/benchmarks.py
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,16 @@ def compute_pose_map(self, results_objects):
symmetric_kpts=[(0, 4), (1, 3)],
)

def _validate_predictions(self, name: str, predictions: dict) -> dict:
"""Fixes filenames for predictions made on old versions of the dataset"""
return super()._validate_predictions(
name,
{
k.replace("Dummy", "D").replace("Dead pup", "DP"): v
for k, v in predictions.items()
},
)


class MarmosetBenchmark(deeplabcut.benchmark.base.Benchmark):
"""Dataset with two marmosets.
Expand Down