Visitar URL original
Fix: read video meta pickles across OSes by C-Achard · Pull Request #3565 · DeepLabCut/DeepLabCut · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions deeplabcut/utils/auxfun_multianimal.py
Original file line number Diff line number Diff line change
Expand Up @@ -211,8 +211,7 @@ def LoadFullMultiAnimalData(dataname):
data = pickle.load(handle)
except (pickle.UnpicklingError, FileNotFoundError):
data = shelve.open(data_file, flag="r")
with metadata_file.open("rb") as handle:
metadata = pickle.load(handle)
metadata = auxiliaryfunctions.read_pickle(metadata_file)
return data, metadata


Expand Down
24 changes: 21 additions & 3 deletions deeplabcut/utils/auxiliaryfunctions.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
import os
import pickle
from collections.abc import Sequence
from pathlib import Path
from pathlib import Path, PurePosixPath, PureWindowsPath

import pandas as pd

Expand Down Expand Up @@ -139,10 +139,28 @@ def attempt_to_make_folder(foldername, recursive=False):
foldername.mkdir(exist_ok=True, parents=True)


_FOREIGN_PATH_NAME, _FOREIGN_PATH_TYPE = (
("PosixPath", PurePosixPath) if os.name == "nt" else ("WindowsPath", PureWindowsPath)
)


class _PortableUnpickler(pickle.Unpickler):
"""Unpickler loading paths pickled on another OS as pure paths."""

def find_class(self, module, name):
# Python 3.13+ pickles paths from ``pathlib._local``
if module in ("pathlib", "pathlib._local") and name == _FOREIGN_PATH_NAME:
return _FOREIGN_PATH_TYPE
return super().find_class(module, name)


def read_pickle(filename):
"""Read the pickle file."""
"""Read the pickle file.

Paths pickled on another OS are loaded as ``PurePosixPath`` or ``PureWindowsPath``.
"""
with Path(filename).open("rb") as handle:
return pickle.load(handle)
return _PortableUnpickler(handle).load()


def write_pickle(filename, data):
Expand Down
25 changes: 24 additions & 1 deletion tests/test_auxiliaryfunctions.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@
#
# Licensed under GNU Lesser General Public License v3.0
#
from pathlib import Path
import os
from pathlib import Path, PurePosixPath, PureWindowsPath
from unittest.mock import patch

import pytest
Expand Down Expand Up @@ -349,3 +350,25 @@ def test_project_path_updated_when_moved(self, tmp_path):

expected = project_dir.absolute()
assert cfg["project_path"] == expected


def _write_path_pickle(path: Path, module: str, cls_name: str) -> Path:
# Protocol 0 pickle of ``{"project_path": <module>.<cls_name>("a", "b")}``; written by hand
# because a concrete path of the other OS cannot be created to pickle it
path.write_bytes(f"(dVproject_path\nc{module}\n{cls_name}\n(Va\nVb\ntRs.".encode())
return path


@pytest.mark.parametrize("module", ["pathlib", "pathlib._local"])
def test_read_pickle_loads_foreign_paths_as_pure(tmp_path, module):
"""Paths pickled on another OS load as pure paths instead of raising (#3562)."""
foreign_name, pure_type = ("PosixPath", PurePosixPath) if os.name == "nt" else ("WindowsPath", PureWindowsPath)
data = auxiliaryfunctions.read_pickle(_write_path_pickle(tmp_path / "meta.pickle", module, foreign_name))
assert type(data["project_path"]) is pure_type
assert data["project_path"].parts == ("a", "b")


def test_read_pickle_keeps_native_paths(tmp_path):
native_type = type(Path())
data = auxiliaryfunctions.read_pickle(_write_path_pickle(tmp_path / "meta.pickle", "pathlib", native_type.__name__))
assert type(data["project_path"]) is native_type
Loading