Visitar URL original
Harden path-handling after pathlib migration by deruyter92 · Pull Request #3421 · DeepLabCut/DeepLabCut · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
deprecate str-based grab_files_in_folder and migrate callers to p…
…athlib `Path`

- Deprecate `grab_files_in_folder` (replaced by `collect_video_paths`)
- Replace all internal callers with `collect_video_paths`
- Migrate `check_if_not_analyzed`, `save_data`, `_convert_h5_files_to`,
  and `find_analyzed_data` from `str`-based path ops to pathlib (.stem,
  .with_suffix, .with_name, .removeprefix)
- Remove redundant `Path()` wraps and `str()` conversions at call sites
  • Loading branch information
deruyter92 committed Jul 21, 2026
commit a272894ddf7732018ea48c973a5043547533391b
10 changes: 4 additions & 6 deletions deeplabcut/gui/tabs/create_project.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
)
from deeplabcut.gui.widgets import ClickableLabel, ItemSelectionFrame
from deeplabcut.utils import auxiliaryfunctions
from deeplabcut.utils.auxfun_videos import collect_video_paths


class DynamicTextList(QtWidgets.QWidget):
Expand Down Expand Up @@ -434,12 +435,9 @@ def browse_videos(self):
if not folder:
return

for video in auxiliaryfunctions.grab_files_in_folder(
folder,
relative=False,
):
if Path(video).suffix[1:].lower() in DLCParams.VIDEOTYPES[1:]:
self.video_frame.fancy_list.add_item(video)
for video in collect_video_paths(folder):
if video.suffix[1:].lower() in DLCParams.VIDEOTYPES[1:]:
self.video_frame.fancy_list.add_item(str(video))

def finalize_project(self):
fields = [self.proj_line, self.exp_line]
Expand Down
8 changes: 4 additions & 4 deletions deeplabcut/pose_estimation_tensorflow/predict_videos.py
Original file line number Diff line number Diff line change
Expand Up @@ -1017,7 +1017,7 @@ def AnalyzeVideo(
metadata = {"data": dictionary}

print(f"Saving results in {destfolder}...")
dataname = str(Path(destfolder) / (vname + DLCscorer + ".h5"))
dataname = Path(destfolder) / (vname + DLCscorer + ".h5")
auxiliaryfunctions.save_data(
PredictedData[:nframes, :],
metadata,
Expand Down Expand Up @@ -1572,15 +1572,15 @@ def convert_detections2tracklets(
destfolder = videofolder
auxiliaryfunctions.attempt_to_make_folder(destfolder)
vname = Path(video).stem
dataname = str(Path(destfolder) / (vname + DLCscorer + ".h5"))
dataname = Path(destfolder) / (vname + DLCscorer + ".h5")
data, metadata = auxfun_multianimal.LoadFullMultiAnimalData(dataname)
if track_method == "ellipse":
method = "el"
elif track_method == "box":
method = "bx"
else:
method = "sk"
trackname = dataname.split(".h5")[0] + f"_{method}.pickle"
trackname = dataname.with_name(f"{dataname.stem}_{method}.pickle")
# NOTE: If dataname line above is changed then line below is obsolete?
# trackname = trackname.replace(videofolder, destfolder)
if Path(trackname).is_file() and not overwrite: # TODO: check if metadata are identical (same parameters!)
Expand Down Expand Up @@ -1638,7 +1638,7 @@ def convert_detections2tracklets(
identity_only=identity_only,
min_n_links=inferencecfg["minimalnumberofconnections"],
)
assemblies_filename = dataname.split(".h5")[0] + "_assemblies.pickle"
assemblies_filename = dataname.with_name(dataname.stem + "_assemblies.pickle")
if not Path(assemblies_filename).exists() or overwrite:
if calibrate:
trainingsetfolder = auxiliaryfunctions.get_training_set_folder(cfg)
Expand Down
30 changes: 15 additions & 15 deletions deeplabcut/utils/auxiliaryfunctions.py
Original file line number Diff line number Diff line change
Expand Up @@ -169,12 +169,13 @@ def save_data(PredicteData, metadata, dataname, pdindex, imagenames, save_as_csv
"""Save predicted data as h5 file and metadata as pickle file; created by
predict_videos.py.
"""
dataname = Path(dataname)
DataMachine = pd.DataFrame(PredicteData, columns=pdindex, index=imagenames)
if save_as_csv:
print("Saving csv poses!")
DataMachine.to_csv(dataname.split(".h5")[0] + ".csv")
DataMachine.to_csv(dataname.with_suffix(".csv"))
DataMachine.to_hdf(dataname, key="df_with_missing", format="table", mode="w")
with Path(dataname.split(".h5")[0] + "_meta.pickle").open("wb") as f:
with dataname.with_name(dataname.stem + "_meta.pickle").open("wb") as f:
# Pickle the 'data' dictionary using the highest protocol available.
pickle.dump(metadata, f, pickle.HIGHEST_PROTOCOL)

Expand Down Expand Up @@ -202,8 +203,7 @@ def load_metadata(metadatafile: str | Path):
return trainingdata_details, trainIndices, testIndices, testFraction_data


# TODO: @deruyter92 2026-05-20: this function could be updated to match the
# signature of collect_video_paths, allowing for multiple extensions.
@deprecated(replacement="deeplabcut.collect_video_paths", since="3.0.1")
def grab_files_in_folder(folder, ext="", relative=True):
"""Return the paths of files with extension *ext* present in *folder*."""
for file in Path(folder).iterdir():
Expand Down Expand Up @@ -559,28 +559,28 @@ def check_if_post_processing(folder, vname, DLCscorer, DLCscorerlegacy, suffix="


def check_if_not_analyzed(destfolder, vname, DLCscorer, DLCscorerlegacy, flag="video"):
h5files = list(grab_files_in_folder(destfolder, "h5", relative=False))
h5files = collect_video_paths(destfolder, extensions=".h5")
if not len(h5files):
dataname = str(Path(destfolder) / (vname + DLCscorer + ".h5"))
dataname = Path(destfolder) / (vname + DLCscorer + ".h5")
return True, dataname, DLCscorer

# Iterate over data files and stop as soon as one matching the scorer is found
for h5file in h5files:
if vname + DLCscorer in Path(h5file).stem:
if vname + DLCscorer in h5file.stem:
if flag == "video":
print("Video already analyzed!", h5file)
elif flag == "framestack":
print("Frames already analyzed!", h5file)
return False, h5file, DLCscorer
elif vname + DLCscorerlegacy in Path(h5file).stem:
elif vname + DLCscorerlegacy in h5file.stem:
if flag == "video":
print("Video already analyzed!", h5file)
elif flag == "framestack":
print("Frames already analyzed!", h5file)
return False, h5file, DLCscorerlegacy

# If there was no match...
dataname = str(Path(destfolder) / (vname + DLCscorer + ".h5"))
dataname = Path(destfolder) / (vname + DLCscorer + ".h5")
return True, dataname, DLCscorer


Expand Down Expand Up @@ -640,19 +640,19 @@ def find_analyzed_data(folder, videoname: str, scorer: str, filtered=False, trac
tracker = TRACK_METHODS.get(track_method, "")

candidates = []
for file in grab_files_in_folder(folder, "h5"):
stem = Path(file).stem.replace("_filtered", "")
starts_by_scorer = Path(file).name.startswith((videoname + scorer, videoname + scorer_legacy))
for file in collect_video_paths(folder, extensions=".h5"):
stem = file.stem.replace("_filtered", "")
starts_by_scorer = file.name.startswith((videoname + scorer, videoname + scorer_legacy))
if tracker:
matches_tracker = stem.endswith(tracker)
else:
matches_tracker = not any(stem.endswith(s) for s in TRACK_METHODS.values())
if all(
(
starts_by_scorer,
"skeleton" not in file,
"skeleton" not in file.name,
matches_tracker,
(filtered and "filtered" in file) or (not filtered and "filtered" not in file),
(filtered and "filtered" in file.name) or (not filtered and "filtered" not in file.name),
)
):
candidates.append(file)
Expand All @@ -670,7 +670,7 @@ def find_analyzed_data(folder, videoname: str, scorer: str, filtered=False, trac
n_candidates = len(candidates)
if n_candidates > 1: # This should not be happening anyway...
print(f"{n_candidates} possible data files were found: {candidates}.\nPicking the first by default...")
filepath = str(Path(folder) / candidates[0])
filepath = str(candidates[0])
scorer = scorer if scorer in filepath else scorer_legacy
return filepath, scorer, suffix

Expand Down
34 changes: 15 additions & 19 deletions deeplabcut/utils/conversioncode.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

import deeplabcut as dlc
from deeplabcut.utils import auxiliaryfunctions
from deeplabcut.utils.auxfun_videos import collect_video_paths

SUPPORTED_FILETYPES = "csv", "nwb"

Expand Down Expand Up @@ -207,8 +208,7 @@ def adapt_labeled_data_to_new_project(
convertcsv2h5(config_path, userfeedback=userfeedback)


# TODO: @deruyter92 2026-05-20: this function still uses grab_files_in_folder instead
# of collect_video_paths and videotype instead of video_extensions.
# TODO: @deruyter92 2026-05-20: this function uses videotype instead of video_extensions.
def analyze_videos_converth5_to_csv(video_folder, videotype=".mp4", listofvideos=False):
"""By default the output poses (when running analyze_videos) are stored as
MultiIndex Pandas Array, which contains the name of the network, body part name, (x,
Expand All @@ -234,18 +234,17 @@ def analyze_videos_converth5_to_csv(video_folder, videotype=".mp4", listofvideos
if listofvideos: # can also be called with a list of videos (from GUI)
videos = video_folder # GUI gives a list of videos
if len(videos) > 0:
h5_files = list(auxiliaryfunctions.grab_files_in_folder(Path(videos[0]).parent, "h5", relative=False))
h5_files = collect_video_paths(Path(videos[0]).parent, extensions=".h5")
else:
h5_files = []
else:
h5_files = list(auxiliaryfunctions.grab_files_in_folder(video_folder, "h5", relative=False))
videos = auxiliaryfunctions.grab_files_in_folder(video_folder, videotype, relative=False)
h5_files = collect_video_paths(video_folder, extensions=".h5")
videos = collect_video_paths(video_folder, extensions=videotype)

_convert_h5_files_to("csv", None, h5_files, videos)


# TODO: @deruyter92 2026-05-20: this function still uses grab_files_in_folder instead
# of collect_video_paths and videotype instead of video_extensions.
# TODO: @deruyter92 2026-05-20: this function uses videotype instead of video_extensions.
def analyze_videos_converth5_to_nwb(
config: str | Path,
video_folder: str | Path,
Expand All @@ -272,12 +271,12 @@ def analyze_videos_converth5_to_nwb(
if listofvideos: # can also be called with a list of videos (from GUI)
videos = video_folder # GUI gives a list of videos
if len(videos) > 0:
h5_files = list(auxiliaryfunctions.grab_files_in_folder(Path(videos[0]).parent, "h5", relative=False))
h5_files = collect_video_paths(Path(videos[0]).parent, extensions=".h5")
else:
h5_files = []
else:
h5_files = list(auxiliaryfunctions.grab_files_in_folder(video_folder, "h5", relative=False))
videos = auxiliaryfunctions.grab_files_in_folder(video_folder, videotype, relative=False)
h5_files = collect_video_paths(video_folder, extensions=".h5")
videos = collect_video_paths(video_folder, extensions=videotype)

_convert_h5_files_to("nwb", config, h5_files, videos)

Expand All @@ -297,18 +296,18 @@ def _convert_h5_files_to(filetype, config, h5_files, videos):
raise ImportError("The package `dlc2nwb` is missing. Please run `pip install dlc2nwb`.") from e

for video in videos:
if "_labeled" in video:
if "_labeled" in video.name:
continue
vname = Path(video).stem
vname = video.stem
for file in h5_files:
if vname in file:
scorer = file.split(vname)[1].split(".h5")[0]
if vname in file.name:
scorer = file.stem.removeprefix(vname)
if "DLC" in scorer or "DeepCut" in scorer:
print("Found output file for scorer:", scorer)
print(f"Converting {file}...")
if filetype == "csv":
df = pd.read_hdf(file)
df.to_csv(file.replace(".h5", ".csv"))
df.to_csv(file.with_suffix(".csv"))
else:
convert_h5_to_nwb(config, file)

Expand All @@ -323,10 +322,7 @@ def merge_windowsannotationdataONlinuxsystem(cfg):
"""
AnnotationData = []
data_path = Path(cfg["project_path"], "labeled-data")
annotationfolders = []
for elem in auxiliaryfunctions.grab_files_in_folder(data_path, relative=False):
if Path(elem).is_dir():
annotationfolders.append(elem)
annotationfolders = [d for d in data_path.iterdir() if d.is_dir()]
print("The following folders were found:", annotationfolders)
for folder in annotationfolders:
filename = str(Path(folder) / ("CollectedData_" + cfg["scorer"] + ".h5"))
Expand Down