Visitar URL original
Fix CSVLogger: read existing learning_stats.csv file when resuming previous training by deruyter92 · Pull Request #3177 · DeepLabCut/DeepLabCut · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Fix CSVLogger reads existing file for resumed training
Previously, when resuming training the CSVLogger created a new learning_stats.csv file for the second training run, overwriting previously saved training metrics.

This commit addresses feature request #3176 and changes the behaviour to reading any previously saved log metrics and appending new epochs to the learning_stats.csv file.
  • Loading branch information
deruyter92 committed Jan 13, 2026
commit a78f75d19710db012769d102c42d0c6dd1d95c39
52 changes: 51 additions & 1 deletion deeplabcut/pose_estimation_pytorch/runners/logger.py
Original file line number Diff line number Diff line change
Expand Up @@ -413,6 +413,10 @@ def __init__(self, train_folder: str, log_filename: str) -> None:
self._metric_store: list[dict] = []
self._logged_metrics: set[str] = set()

# Load existing data if the file exists (e.g., when resuming from snapshot)
if self.log_file.exists():
self._load_existing_data()

def log(self, metrics: dict[str, Any], step: Optional[int] = None) -> None:
"""Logs metrics from runs

Expand Down Expand Up @@ -450,6 +454,47 @@ def log_config(self, config: dict = None) -> None:
"""
pass

def _load_existing_data(self) -> None:
"""Loads existing CSV data if the log file exists"""
try:
with open(self.log_file, "r", newline="") as f:
reader = csv.DictReader(f)

Comment thread
deruyter92 marked this conversation as resolved.
Resolved
# Update logged metrics from header
if "step" not in reader.fieldnames:
return # Invalid format

Comment thread
deruyter92 marked this conversation as resolved.
Resolved
metric_names = [m for m in reader.fieldnames if m != "step"]
self._logged_metrics.update(metric_names)

# Load data rows
for row in reader:
try:
step = int(row["step"])
except (ValueError, KeyError):
continue

Comment thread
deruyter92 marked this conversation as resolved.
Resolved
# Convert metric values: empty strings -> None, numeric strings -> float
step_metrics = {}
for metric in metric_names:
value = row.get(metric, "").strip()
if not value:
step_metrics[metric] = None
else:
try:
step_metrics[metric] = float(value)
except ValueError:
step_metrics[metric] = value

Comment thread
deruyter92 marked this conversation as resolved.
Resolved
self._steps.append(step)
self._metric_store.append(step_metrics)

Comment thread
deruyter92 marked this conversation as resolved.
Resolved
except Exception as e:
logging.warning(
f"Failed to load existing CSV data from {self.log_file}: {e}. "
"Starting with empty log."
)

def _prepare_logs(self) -> list[list]:
"""Prepares the data to log as a list of strings"""
if len(self._metric_store) == 0:
Expand All @@ -458,6 +503,11 @@ def _prepare_logs(self) -> list[list]:
metrics = list(sorted(self._logged_metrics))
logs = [["step"] + metrics]
for step, step_metrics in zip(self._steps, self._metric_store):
logs.append([step] + [step_metrics.get(m) for m in metrics])
# Convert None values to empty strings for proper CSV formatting
row = [step] + [
"" if step_metrics.get(m) is None else step_metrics.get(m)
for m in metrics
]
logs.append(row)

return logs
26 changes: 26 additions & 0 deletions tests/pose_estimation_pytorch/runners/test_logger.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
# Licensed under GNU Lesser General Public License v3.0
#
"""Tests loggers"""
from pathlib import Path
from typing import Any

import pytest
Expand Down Expand Up @@ -73,3 +74,28 @@ def test_prepare_image(keypoints: list[list[float]], denormalize: bool) -> None:
keypoints=keypoints,
bboxes=None,
)


def test_csv_logger_resume(tmp_path: Path) -> None:
"""Test CSVLogger preserves data when resuming from snapshot"""
log_file = tmp_path / "learning_stats.csv"

# Initial training: log some metrics
logger1 = logging.CSVLogger(str(tmp_path), "learning_stats.csv")
logger1.log({"loss": 0.5, "accuracy": 0.8}, step=1)
logger1.log({"loss": 0.4, "accuracy": 0.9}, step=2)

assert log_file.exists()
assert len(logger1._steps) == 2

# Resume training: should load existing data
logger2 = logging.CSVLogger(str(tmp_path), "learning_stats.csv")
assert len(logger2._steps) == 2
assert logger2._steps == [1, 2]
assert logger2._metric_store[0]["loss"] == 0.5
assert logger2._metric_store[1]["accuracy"] == 0.9

Comment thread
deruyter92 marked this conversation as resolved.
Resolved
Comment thread
deruyter92 marked this conversation as resolved.
Resolved
# Log new data: should append, not overwrite
logger2.log({"loss": 0.3, "accuracy": 0.95}, step=3)
assert len(logger2._steps) == 3
assert logger2._steps == [1, 2, 3]
Loading